Coverage for tests/test_executors.py: 16%
338 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-28 09:38 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-28 09:38 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import faulthandler
26import logging
27import signal
28import sys
29import time
30import unittest
31import warnings
32from multiprocessing import Manager
34import networkx as nx
35import psutil
36from lsst.ctrl.mpexec import (
37 ExecutionStatus,
38 MPGraphExecutor,
39 MPGraphExecutorError,
40 MPTimeoutError,
41 QuantumExecutor,
42 QuantumReport,
43)
44from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
45from lsst.pipe.base import NodeId
47logging.basicConfig(level=logging.DEBUG)
49_LOG = logging.getLogger(__name__)
52class QuantumExecutorMock(QuantumExecutor):
53 """Mock class for QuantumExecutor"""
55 def __init__(self, mp=False):
56 self.quanta = []
57 if mp:
58 # in multiprocess mode use shared list
59 manager = Manager()
60 self.quanta = manager.list()
61 self.report = None
62 self._execute_called = False
64 def execute(self, taskDef, quantum):
65 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
66 self._execute_called = True
67 if taskDef.taskClass:
68 try:
69 # only works for one of the TaskMock classes below
70 taskDef.taskClass().runQuantum()
71 self.report = QuantumReport(dataId=quantum.dataId, taskLabel=taskDef.label)
72 except Exception as exc:
73 self.report = QuantumReport.from_exception(
74 exception=exc,
75 dataId=quantum.dataId,
76 taskLabel=taskDef.label,
77 )
78 raise
79 self.quanta.append(quantum)
80 return quantum
82 def getReport(self):
83 if not self._execute_called:
84 raise RuntimeError("getReport called before execute")
85 return self.report
87 def getDataIds(self, field):
88 """Returns values for dataId field for each visited quanta"""
89 return [quantum.dataId[field] for quantum in self.quanta]
92class QuantumMock:
93 def __init__(self, dataId):
94 self.dataId = dataId
96 def __eq__(self, other):
97 return self.dataId == other.dataId
99 def __hash__(self):
100 # dict.__eq__ is order-insensitive
101 return hash(tuple(sorted(kv for kv in self.dataId.items())))
104class QuantumIterDataMock:
105 """Simple class to mock QuantumIterData."""
107 def __init__(self, index, taskDef, **dataId):
108 self.index = index
109 self.taskDef = taskDef
110 self.quantum = QuantumMock(dataId)
111 self.dependencies = set()
112 self.nodeId = NodeId(index, "DummyBuildString")
115class QuantumGraphMock:
116 """Mock for quantum graph."""
118 def __init__(self, qdata):
119 self._graph = nx.DiGraph()
120 previous = qdata[0]
121 for node in qdata[1:]:
122 self._graph.add_edge(previous, node)
123 previous = node
125 def __iter__(self):
126 yield from nx.topological_sort(self._graph)
128 def __len__(self):
129 return len(self._graph)
131 def findTaskDefByLabel(self, label):
132 for q in self:
133 if q.taskDef.label == label:
134 return q.taskDef
136 def getQuantaForTask(self, taskDef):
137 nodes = self.getNodesForTask(taskDef)
138 return {q.quantum for q in nodes}
140 def getNodesForTask(self, taskDef):
141 quanta = set()
142 for q in self:
143 if q.taskDef == taskDef:
144 quanta.add(q)
145 return quanta
147 @property
148 def graph(self):
149 return self._graph
151 def findCycle(self):
152 return []
154 def determineInputsToQuantumNode(self, node):
155 result = set()
156 for n in node.dependencies:
157 for otherNode in self:
158 if otherNode.index == n:
159 result.add(otherNode)
160 return result
163class TaskMockMP:
164 """Simple mock class for task supporting multiprocessing."""
166 canMultiprocess = True
168 def runQuantum(self):
169 _LOG.debug("TaskMockMP.runQuantum")
170 pass
173class TaskMockFail:
174 """Simple mock class for task which fails."""
176 canMultiprocess = True
178 def runQuantum(self):
179 _LOG.debug("TaskMockFail.runQuantum")
180 raise ValueError("expected failure")
183class TaskMockCrash:
184 """Simple mock class for task which fails."""
186 canMultiprocess = True
188 def runQuantum(self):
189 _LOG.debug("TaskMockCrash.runQuantum")
190 # Disable fault handler to suppress long scary traceback.
191 faulthandler.disable()
192 signal.raise_signal(signal.SIGILL)
195class TaskMockLongSleep:
196 """Simple mock class for task which "runs" for very long time."""
198 canMultiprocess = True
200 def runQuantum(self):
201 _LOG.debug("TaskMockLongSleep.runQuantum")
202 time.sleep(100.0)
205class TaskMockNoMP:
206 """Simple mock class for task not supporting multiprocessing."""
208 canMultiprocess = False
211class TaskDefMock:
212 """Simple mock class for task definition in a pipeline."""
214 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
215 self.taskName = taskName
216 self.config = config
217 self.taskClass = taskClass
218 self.label = label
220 def __str__(self):
221 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
224def _count_status(report, status):
225 """Count number of quanta witha a given status."""
226 return len([qrep for qrep in report.quantaReports if qrep.status is status])
229class MPGraphExecutorTestCase(unittest.TestCase):
230 """A test case for MPGraphExecutor class"""
232 def test_mpexec_nomp(self):
233 """Make simple graph and execute"""
235 taskDef = TaskDefMock()
236 qgraph = QuantumGraphMock(
237 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
238 )
240 # run in single-process mode
241 qexec = QuantumExecutorMock()
242 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
243 mpexec.execute(qgraph)
244 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
245 report = mpexec.getReport()
246 self.assertEqual(report.status, ExecutionStatus.SUCCESS)
247 self.assertIsNone(report.exitCode)
248 self.assertIsNone(report.exceptionInfo)
249 self.assertEqual(len(report.quantaReports), 3)
250 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports))
251 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports))
252 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
253 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports))
255 def test_mpexec_mp(self):
256 """Make simple graph and execute"""
258 taskDef = TaskDefMock()
259 qgraph = QuantumGraphMock(
260 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
261 )
263 methods = ["spawn"]
264 if sys.platform == "linux":
265 methods.append("fork")
266 methods.append("forkserver")
268 for method in methods:
269 with self.subTest(startMethod=method):
270 # Run in multi-process mode, the order of results is not
271 # defined.
272 qexec = QuantumExecutorMock(mp=True)
273 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method)
274 mpexec.execute(qgraph)
275 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
276 report = mpexec.getReport()
277 self.assertEqual(report.status, ExecutionStatus.SUCCESS)
278 self.assertIsNone(report.exitCode)
279 self.assertIsNone(report.exceptionInfo)
280 self.assertEqual(len(report.quantaReports), 3)
281 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports))
282 self.assertTrue(all(qrep.exitCode == 0 for qrep in report.quantaReports))
283 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
284 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports))
286 def test_mpexec_nompsupport(self):
287 """Try to run MP for task that has no MP support which should fail"""
289 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
290 qgraph = QuantumGraphMock(
291 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
292 )
294 # run in multi-process mode
295 qexec = QuantumExecutorMock()
296 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
297 with self.assertRaisesRegex(MPGraphExecutorError, "Task Task does not support multiprocessing"):
298 mpexec.execute(qgraph)
300 def test_mpexec_fixup(self):
301 """Make simple graph and execute, add dependencies by executing fixup
302 code.
303 """
305 taskDef = TaskDefMock()
307 for reverse in (False, True):
308 qgraph = QuantumGraphMock(
309 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
310 )
312 qexec = QuantumExecutorMock()
313 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
314 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, executionGraphFixup=fixup)
315 mpexec.execute(qgraph)
317 expected = [0, 1, 2]
318 if reverse:
319 expected = list(reversed(expected))
320 self.assertEqual(qexec.getDataIds("detector"), expected)
322 def test_mpexec_timeout(self):
323 """Fail due to timeout"""
325 taskDef = TaskDefMock()
326 taskDefSleep = TaskDefMock(taskClass=TaskMockLongSleep)
327 qgraph = QuantumGraphMock(
328 [
329 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
330 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
331 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
332 ]
333 )
335 # with failFast we'll get immediate MPTimeoutError
336 qexec = QuantumExecutorMock(mp=True)
337 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
338 with self.assertRaises(MPTimeoutError):
339 mpexec.execute(qgraph)
340 report = mpexec.getReport()
341 self.assertEqual(report.status, ExecutionStatus.TIMEOUT)
342 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError")
343 self.assertGreater(len(report.quantaReports), 0)
344 self.assertEqual(_count_status(report, ExecutionStatus.TIMEOUT), 1)
345 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports))
346 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
348 # with failFast=False exception happens after last task finishes
349 qexec = QuantumExecutorMock(mp=True)
350 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
351 with self.assertRaises(MPTimeoutError):
352 mpexec.execute(qgraph)
353 # We expect two tasks (0 and 2) to finish successfully and one task to
354 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
355 # finish on time, so expect more timeouts and issue a warning.
356 detectorIds = set(qexec.getDataIds("detector"))
357 self.assertLess(len(detectorIds), 3)
358 if detectorIds != {0, 2}:
359 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
360 report = mpexec.getReport()
361 self.assertEqual(report.status, ExecutionStatus.TIMEOUT)
362 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError")
363 self.assertGreater(len(report.quantaReports), 0)
364 self.assertGreater(_count_status(report, ExecutionStatus.TIMEOUT), 0)
365 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports))
366 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
368 def test_mpexec_failure(self):
369 """Failure in one task should not stop other tasks"""
371 taskDef = TaskDefMock()
372 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
373 qgraph = QuantumGraphMock(
374 [
375 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
376 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
377 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
378 ]
379 )
381 qexec = QuantumExecutorMock(mp=True)
382 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
383 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
384 mpexec.execute(qgraph)
385 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
386 report = mpexec.getReport()
387 self.assertEqual(report.status, ExecutionStatus.FAILURE)
388 self.assertEqual(
389 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
390 )
391 self.assertGreater(len(report.quantaReports), 0)
392 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
393 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
394 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
395 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
397 def test_mpexec_failure_dep(self):
398 """Failure in one task should skip dependents"""
400 taskDef = TaskDefMock()
401 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
402 qdata = [
403 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
404 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
405 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
406 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
407 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
408 ]
409 qdata[2].dependencies.add(1)
410 qdata[4].dependencies.add(3)
411 qdata[4].dependencies.add(2)
413 qgraph = QuantumGraphMock(qdata)
415 qexec = QuantumExecutorMock(mp=True)
416 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
417 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
418 mpexec.execute(qgraph)
419 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
420 report = mpexec.getReport()
421 self.assertEqual(report.status, ExecutionStatus.FAILURE)
422 self.assertEqual(
423 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
424 )
425 # Dependencies of failed tasks do not appear in quantaReports
426 self.assertGreater(len(report.quantaReports), 0)
427 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
428 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
429 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2)
430 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
431 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
433 def test_mpexec_failure_dep_nomp(self):
434 """Failure in one task should skip dependents, in-process version"""
436 taskDef = TaskDefMock()
437 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
438 qdata = [
439 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
440 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
441 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
442 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
443 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
444 ]
445 qdata[2].dependencies.add(1)
446 qdata[4].dependencies.add(3)
447 qdata[4].dependencies.add(2)
449 qgraph = QuantumGraphMock(qdata)
451 qexec = QuantumExecutorMock()
452 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
453 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
454 mpexec.execute(qgraph)
455 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
456 report = mpexec.getReport()
457 self.assertEqual(report.status, ExecutionStatus.FAILURE)
458 self.assertEqual(
459 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
460 )
461 # Dependencies of failed tasks do not appear in quantaReports
462 self.assertGreater(len(report.quantaReports), 0)
463 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
464 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
465 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2)
466 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports))
467 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
469 def test_mpexec_failure_failfast(self):
470 """Fast fail stops quickly.
472 Timing delay of task #3 should be sufficient to process
473 failure and raise exception.
474 """
476 taskDef = TaskDefMock()
477 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
478 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
479 qdata = [
480 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
481 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
482 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
483 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
484 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
485 ]
486 qdata[1].dependencies.add(0)
487 qdata[2].dependencies.add(1)
488 qdata[4].dependencies.add(3)
489 qdata[4].dependencies.add(2)
491 qgraph = QuantumGraphMock(qdata)
493 qexec = QuantumExecutorMock(mp=True)
494 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
495 with self.assertRaisesRegex(MPGraphExecutorError, "failed, exit code=1"):
496 mpexec.execute(qgraph)
497 self.assertCountEqual(qexec.getDataIds("detector"), [0])
498 report = mpexec.getReport()
499 self.assertEqual(report.status, ExecutionStatus.FAILURE)
500 self.assertEqual(
501 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
502 )
503 # Dependencies of failed tasks do not appear in quantaReports
504 self.assertGreater(len(report.quantaReports), 0)
505 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
506 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
507 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
509 def test_mpexec_crash(self):
510 """Check task crash due to signal"""
512 taskDef = TaskDefMock()
513 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash)
514 qgraph = QuantumGraphMock(
515 [
516 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
517 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1),
518 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
519 ]
520 )
522 qexec = QuantumExecutorMock(mp=True)
523 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
524 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
525 mpexec.execute(qgraph)
526 report = mpexec.getReport()
527 self.assertEqual(report.status, ExecutionStatus.FAILURE)
528 self.assertEqual(
529 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
530 )
531 # Dependencies of failed tasks do not appear in quantaReports
532 self.assertGreater(len(report.quantaReports), 0)
533 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
534 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
535 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports))
536 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
538 def test_mpexec_crash_failfast(self):
539 """Check task crash due to signal with --fail-fast"""
541 taskDef = TaskDefMock()
542 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash)
543 qgraph = QuantumGraphMock(
544 [
545 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
546 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1),
547 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
548 ]
549 )
551 qexec = QuantumExecutorMock(mp=True)
552 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
553 with self.assertRaisesRegex(MPGraphExecutorError, "failed, killed by signal 4 .Illegal instruction"):
554 mpexec.execute(qgraph)
555 report = mpexec.getReport()
556 self.assertEqual(report.status, ExecutionStatus.FAILURE)
557 self.assertEqual(
558 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
559 )
560 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
561 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports))
562 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
564 def test_mpexec_num_fd(self):
565 """Check that number of open files stays reasonable"""
567 taskDef = TaskDefMock()
568 qgraph = QuantumGraphMock(
569 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)]
570 )
572 this_proc = psutil.Process()
573 num_fds_0 = this_proc.num_fds()
575 # run in multi-process mode, the order of results is not defined
576 qexec = QuantumExecutorMock(mp=True)
577 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
578 mpexec.execute(qgraph)
580 num_fds_1 = this_proc.num_fds()
581 # They should be the same but allow small growth just in case.
582 # Without DM-26728 fix the difference would be equal to number of
583 # quanta (20).
584 self.assertLess(num_fds_1 - num_fds_0, 5)
587if __name__ == "__main__": 587 ↛ 588line 587 didn't jump to line 588, because the condition on line 587 was never true
588 unittest.main()