Coverage for tests/test_executors.py: 19%

336 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-07-03 01:28 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import signal 

27import sys 

28import time 

29import unittest 

30import warnings 

31from multiprocessing import Manager 

32 

33import networkx as nx 

34import psutil 

35from lsst.ctrl.mpexec import ( 

36 ExecutionStatus, 

37 MPGraphExecutor, 

38 MPGraphExecutorError, 

39 MPTimeoutError, 

40 QuantumExecutor, 

41 QuantumReport, 

42) 

43from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

44from lsst.pipe.base import NodeId 

45 

46logging.basicConfig(level=logging.DEBUG) 

47 

48_LOG = logging.getLogger(__name__) 

49 

50 

51class QuantumExecutorMock(QuantumExecutor): 

52 """Mock class for QuantumExecutor""" 

53 

54 def __init__(self, mp=False): 

55 self.quanta = [] 

56 if mp: 

57 # in multiprocess mode use shared list 

58 manager = Manager() 

59 self.quanta = manager.list() 

60 self.report = None 

61 self._execute_called = False 

62 

63 def execute(self, taskDef, quantum, butler): 

64 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

65 self._execute_called = True 

66 if taskDef.taskClass: 

67 try: 

68 # only works for one of the TaskMock classes below 

69 taskDef.taskClass().runQuantum() 

70 self.report = QuantumReport(dataId=quantum.dataId, taskLabel=taskDef.label) 

71 except Exception as exc: 

72 self.report = QuantumReport.from_exception( 

73 exception=exc, 

74 dataId=quantum.dataId, 

75 taskLabel=taskDef.label, 

76 ) 

77 raise 

78 self.quanta.append(quantum) 

79 return quantum 

80 

81 def getReport(self): 

82 if not self._execute_called: 

83 raise RuntimeError("getReport called before execute") 

84 return self.report 

85 

86 def getDataIds(self, field): 

87 """Returns values for dataId field for each visited quanta""" 

88 return [quantum.dataId[field] for quantum in self.quanta] 

89 

90 

91class QuantumMock: 

92 def __init__(self, dataId): 

93 self.dataId = dataId 

94 

95 def __eq__(self, other): 

96 return self.dataId == other.dataId 

97 

98 def __hash__(self): 

99 # dict.__eq__ is order-insensitive 

100 return hash(tuple(sorted(kv for kv in self.dataId.items()))) 

101 

102 

103class QuantumIterDataMock: 

104 """Simple class to mock QuantumIterData.""" 

105 

106 def __init__(self, index, taskDef, **dataId): 

107 self.index = index 

108 self.taskDef = taskDef 

109 self.quantum = QuantumMock(dataId) 

110 self.dependencies = set() 

111 self.nodeId = NodeId(index, "DummyBuildString") 

112 

113 

114class QuantumGraphMock: 

115 """Mock for quantum graph.""" 

116 

117 def __init__(self, qdata): 

118 self._graph = nx.DiGraph() 

119 previous = qdata[0] 

120 for node in qdata[1:]: 

121 self._graph.add_edge(previous, node) 

122 previous = node 

123 

124 def __iter__(self): 

125 yield from nx.topological_sort(self._graph) 

126 

127 def __len__(self): 

128 return len(self._graph) 

129 

130 def findTaskDefByLabel(self, label): 

131 for q in self: 

132 if q.taskDef.label == label: 

133 return q.taskDef 

134 

135 def getQuantaForTask(self, taskDef): 

136 nodes = self.getNodesForTask(taskDef) 

137 return {q.quantum for q in nodes} 

138 

139 def getNodesForTask(self, taskDef): 

140 quanta = set() 

141 for q in self: 

142 if q.taskDef == taskDef: 

143 quanta.add(q) 

144 return quanta 

145 

146 @property 

147 def graph(self): 

148 return self._graph 

149 

150 def findCycle(self): 

151 return [] 

152 

153 def determineInputsToQuantumNode(self, node): 

154 result = set() 

155 for n in node.dependencies: 

156 for otherNode in self: 

157 if otherNode.index == n: 

158 result.add(otherNode) 

159 return result 

160 

161 

162class TaskMockMP: 

163 """Simple mock class for task supporting multiprocessing.""" 

164 

165 canMultiprocess = True 

166 

167 def runQuantum(self): 

168 _LOG.debug("TaskMockMP.runQuantum") 

169 pass 

170 

171 

172class TaskMockFail: 

173 """Simple mock class for task which fails.""" 

174 

175 canMultiprocess = True 

176 

177 def runQuantum(self): 

178 _LOG.debug("TaskMockFail.runQuantum") 

179 raise ValueError("expected failure") 

180 

181 

182class TaskMockCrash: 

183 """Simple mock class for task which fails.""" 

184 

185 canMultiprocess = True 

186 

187 def runQuantum(self): 

188 _LOG.debug("TaskMockCrash.runQuantum") 

189 signal.raise_signal(signal.SIGILL) 

190 

191 

192class TaskMockLongSleep: 

193 """Simple mock class for task which "runs" for very long time.""" 

194 

195 canMultiprocess = True 

196 

197 def runQuantum(self): 

198 _LOG.debug("TaskMockLongSleep.runQuantum") 

199 time.sleep(100.0) 

200 

201 

202class TaskMockNoMP: 

203 """Simple mock class for task not supporting multiprocessing.""" 

204 

205 canMultiprocess = False 

206 

207 

208class TaskDefMock: 

209 """Simple mock class for task definition in a pipeline.""" 

210 

211 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

212 self.taskName = taskName 

213 self.config = config 

214 self.taskClass = taskClass 

215 self.label = label 

216 

217 def __str__(self): 

218 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

219 

220 

221def _count_status(report, status): 

222 """Count number of quanta witha a given status.""" 

223 return len([qrep for qrep in report.quantaReports if qrep.status is status]) 

224 

225 

226class MPGraphExecutorTestCase(unittest.TestCase): 

227 """A test case for MPGraphExecutor class""" 

228 

229 def test_mpexec_nomp(self): 

230 """Make simple graph and execute""" 

231 

232 taskDef = TaskDefMock() 

233 qgraph = QuantumGraphMock( 

234 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

235 ) 

236 

237 # run in single-process mode 

238 qexec = QuantumExecutorMock() 

239 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

240 mpexec.execute(qgraph, butler=None) 

241 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

242 report = mpexec.getReport() 

243 self.assertEqual(report.status, ExecutionStatus.SUCCESS) 

244 self.assertIsNone(report.exitCode) 

245 self.assertIsNone(report.exceptionInfo) 

246 self.assertEqual(len(report.quantaReports), 3) 

247 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports)) 

248 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports)) 

249 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

250 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports)) 

251 

252 def test_mpexec_mp(self): 

253 """Make simple graph and execute""" 

254 

255 taskDef = TaskDefMock() 

256 qgraph = QuantumGraphMock( 

257 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

258 ) 

259 

260 methods = ["spawn"] 

261 if sys.platform == "linux": 

262 methods.append("fork") 

263 methods.append("forkserver") 

264 

265 for method in methods: 

266 with self.subTest(startMethod=method): 

267 # Run in multi-process mode, the order of results is not 

268 # defined. 

269 qexec = QuantumExecutorMock(mp=True) 

270 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method) 

271 mpexec.execute(qgraph, butler=None) 

272 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

273 report = mpexec.getReport() 

274 self.assertEqual(report.status, ExecutionStatus.SUCCESS) 

275 self.assertIsNone(report.exitCode) 

276 self.assertIsNone(report.exceptionInfo) 

277 self.assertEqual(len(report.quantaReports), 3) 

278 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports)) 

279 self.assertTrue(all(qrep.exitCode == 0 for qrep in report.quantaReports)) 

280 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

281 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports)) 

282 

283 def test_mpexec_nompsupport(self): 

284 """Try to run MP for task that has no MP support which should fail""" 

285 

286 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

287 qgraph = QuantumGraphMock( 

288 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

289 ) 

290 

291 # run in multi-process mode 

292 qexec = QuantumExecutorMock() 

293 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

294 with self.assertRaisesRegex(MPGraphExecutorError, "Task Task does not support multiprocessing"): 

295 mpexec.execute(qgraph, butler=None) 

296 

297 def test_mpexec_fixup(self): 

298 """Make simple graph and execute, add dependencies by executing fixup 

299 code. 

300 """ 

301 

302 taskDef = TaskDefMock() 

303 

304 for reverse in (False, True): 

305 qgraph = QuantumGraphMock( 

306 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

307 ) 

308 

309 qexec = QuantumExecutorMock() 

310 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

311 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, executionGraphFixup=fixup) 

312 mpexec.execute(qgraph, butler=None) 

313 

314 expected = [0, 1, 2] 

315 if reverse: 

316 expected = list(reversed(expected)) 

317 self.assertEqual(qexec.getDataIds("detector"), expected) 

318 

319 def test_mpexec_timeout(self): 

320 """Fail due to timeout""" 

321 

322 taskDef = TaskDefMock() 

323 taskDefSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

324 qgraph = QuantumGraphMock( 

325 [ 

326 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

327 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

328 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

329 ] 

330 ) 

331 

332 # with failFast we'll get immediate MPTimeoutError 

333 qexec = QuantumExecutorMock(mp=True) 

334 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

335 with self.assertRaises(MPTimeoutError): 

336 mpexec.execute(qgraph, butler=None) 

337 report = mpexec.getReport() 

338 self.assertEqual(report.status, ExecutionStatus.TIMEOUT) 

339 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError") 

340 self.assertGreater(len(report.quantaReports), 0) 

341 self.assertEqual(_count_status(report, ExecutionStatus.TIMEOUT), 1) 

342 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports)) 

343 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

344 

345 # with failFast=False exception happens after last task finishes 

346 qexec = QuantumExecutorMock(mp=True) 

347 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

348 with self.assertRaises(MPTimeoutError): 

349 mpexec.execute(qgraph, butler=None) 

350 # We expect two tasks (0 and 2) to finish successfully and one task to 

351 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

352 # finish on time, so expect more timeouts and issue a warning. 

353 detectorIds = set(qexec.getDataIds("detector")) 

354 self.assertLess(len(detectorIds), 3) 

355 if detectorIds != {0, 2}: 

356 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

357 report = mpexec.getReport() 

358 self.assertEqual(report.status, ExecutionStatus.TIMEOUT) 

359 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError") 

360 self.assertGreater(len(report.quantaReports), 0) 

361 self.assertGreater(_count_status(report, ExecutionStatus.TIMEOUT), 0) 

362 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports)) 

363 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

364 

365 def test_mpexec_failure(self): 

366 """Failure in one task should not stop other tasks""" 

367 

368 taskDef = TaskDefMock() 

369 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

370 qgraph = QuantumGraphMock( 

371 [ 

372 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

373 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

374 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

375 ] 

376 ) 

377 

378 qexec = QuantumExecutorMock(mp=True) 

379 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

380 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

381 mpexec.execute(qgraph, butler=None) 

382 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

383 report = mpexec.getReport() 

384 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

385 self.assertEqual( 

386 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

387 ) 

388 self.assertGreater(len(report.quantaReports), 0) 

389 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1) 

390 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2) 

391 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports)) 

392 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

393 

394 def test_mpexec_failure_dep(self): 

395 """Failure in one task should skip dependents""" 

396 

397 taskDef = TaskDefMock() 

398 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

399 qdata = [ 

400 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

401 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

402 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

403 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

404 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

405 ] 

406 qdata[2].dependencies.add(1) 

407 qdata[4].dependencies.add(3) 

408 qdata[4].dependencies.add(2) 

409 

410 qgraph = QuantumGraphMock(qdata) 

411 

412 qexec = QuantumExecutorMock(mp=True) 

413 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

414 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

415 mpexec.execute(qgraph, butler=None) 

416 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

417 report = mpexec.getReport() 

418 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

419 self.assertEqual( 

420 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

421 ) 

422 # Dependencies of failed tasks do not appear in quantaReports 

423 self.assertGreater(len(report.quantaReports), 0) 

424 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1) 

425 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2) 

426 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2) 

427 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports)) 

428 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

429 

430 def test_mpexec_failure_dep_nomp(self): 

431 """Failure in one task should skip dependents, in-process version""" 

432 

433 taskDef = TaskDefMock() 

434 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

435 qdata = [ 

436 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

437 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

438 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

439 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

440 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

441 ] 

442 qdata[2].dependencies.add(1) 

443 qdata[4].dependencies.add(3) 

444 qdata[4].dependencies.add(2) 

445 

446 qgraph = QuantumGraphMock(qdata) 

447 

448 qexec = QuantumExecutorMock() 

449 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

450 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

451 mpexec.execute(qgraph, butler=None) 

452 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

453 report = mpexec.getReport() 

454 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

455 self.assertEqual( 

456 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

457 ) 

458 # Dependencies of failed tasks do not appear in quantaReports 

459 self.assertGreater(len(report.quantaReports), 0) 

460 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1) 

461 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2) 

462 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2) 

463 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports)) 

464 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

465 

466 def test_mpexec_failure_failfast(self): 

467 """Fast fail stops quickly. 

468 

469 Timing delay of task #3 should be sufficient to process 

470 failure and raise exception. 

471 """ 

472 

473 taskDef = TaskDefMock() 

474 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

475 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

476 qdata = [ 

477 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

478 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

479 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

480 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3), 

481 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

482 ] 

483 qdata[1].dependencies.add(0) 

484 qdata[2].dependencies.add(1) 

485 qdata[4].dependencies.add(3) 

486 qdata[4].dependencies.add(2) 

487 

488 qgraph = QuantumGraphMock(qdata) 

489 

490 qexec = QuantumExecutorMock(mp=True) 

491 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

492 with self.assertRaisesRegex(MPGraphExecutorError, "failed, exit code=1"): 

493 mpexec.execute(qgraph, butler=None) 

494 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

495 report = mpexec.getReport() 

496 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

497 self.assertEqual( 

498 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

499 ) 

500 # Dependencies of failed tasks do not appear in quantaReports 

501 self.assertGreater(len(report.quantaReports), 0) 

502 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1) 

503 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports)) 

504 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

505 

506 def test_mpexec_crash(self): 

507 """Check task crash due to signal""" 

508 

509 taskDef = TaskDefMock() 

510 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash) 

511 qgraph = QuantumGraphMock( 

512 [ 

513 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

514 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1), 

515 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

516 ] 

517 ) 

518 

519 qexec = QuantumExecutorMock(mp=True) 

520 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

521 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

522 mpexec.execute(qgraph, butler=None) 

523 report = mpexec.getReport() 

524 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

525 self.assertEqual( 

526 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

527 ) 

528 # Dependencies of failed tasks do not appear in quantaReports 

529 self.assertGreater(len(report.quantaReports), 0) 

530 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1) 

531 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2) 

532 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports)) 

533 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

534 

535 def test_mpexec_crash_failfast(self): 

536 """Check task crash due to signal with --fail-fast""" 

537 

538 taskDef = TaskDefMock() 

539 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash) 

540 qgraph = QuantumGraphMock( 

541 [ 

542 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

543 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1), 

544 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

545 ] 

546 ) 

547 

548 qexec = QuantumExecutorMock(mp=True) 

549 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

550 with self.assertRaisesRegex(MPGraphExecutorError, "failed, killed by signal 4 .Illegal instruction"): 

551 mpexec.execute(qgraph, butler=None) 

552 report = mpexec.getReport() 

553 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

554 self.assertEqual( 

555 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

556 ) 

557 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1) 

558 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports)) 

559 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

560 

561 def test_mpexec_num_fd(self): 

562 """Check that number of open files stays reasonable""" 

563 

564 taskDef = TaskDefMock() 

565 qgraph = QuantumGraphMock( 

566 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)] 

567 ) 

568 

569 this_proc = psutil.Process() 

570 num_fds_0 = this_proc.num_fds() 

571 

572 # run in multi-process mode, the order of results is not defined 

573 qexec = QuantumExecutorMock(mp=True) 

574 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

575 mpexec.execute(qgraph, butler=None) 

576 

577 num_fds_1 = this_proc.num_fds() 

578 # They should be the same but allow small growth just in case. 

579 # Without DM-26728 fix the difference would be equal to number of 

580 # quanta (20). 

581 self.assertLess(num_fds_1 - num_fds_0, 5) 

582 

583 

584if __name__ == "__main__": 584 ↛ 585line 584 didn't jump to line 585, because the condition on line 584 was never true

585 unittest.main()