Coverage for tests/test_executors.py: 15%
336 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-07 02:42 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-07 02:42 -0800
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26import signal
27import sys
28import time
29import unittest
30import warnings
31from multiprocessing import Manager
33import networkx as nx
34import psutil
35from lsst.ctrl.mpexec import (
36 ExecutionStatus,
37 MPGraphExecutor,
38 MPGraphExecutorError,
39 MPTimeoutError,
40 QuantumExecutor,
41 QuantumReport,
42)
43from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
44from lsst.pipe.base import NodeId
46logging.basicConfig(level=logging.DEBUG)
48_LOG = logging.getLogger(__name__)
51class QuantumExecutorMock(QuantumExecutor):
52 """Mock class for QuantumExecutor"""
54 def __init__(self, mp=False):
55 self.quanta = []
56 if mp:
57 # in multiprocess mode use shared list
58 manager = Manager()
59 self.quanta = manager.list()
60 self.report = None
61 self._execute_called = False
63 def execute(self, taskDef, quantum):
64 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
65 self._execute_called = True
66 if taskDef.taskClass:
67 try:
68 # only works for one of the TaskMock classes below
69 taskDef.taskClass().runQuantum()
70 self.report = QuantumReport(dataId=quantum.dataId, taskLabel=taskDef.label)
71 except Exception as exc:
72 self.report = QuantumReport.from_exception(
73 exception=exc,
74 dataId=quantum.dataId,
75 taskLabel=taskDef.label,
76 )
77 raise
78 self.quanta.append(quantum)
79 return quantum
81 def getReport(self):
82 if not self._execute_called:
83 raise RuntimeError("getReport called before execute")
84 return self.report
86 def getDataIds(self, field):
87 """Returns values for dataId field for each visited quanta"""
88 return [quantum.dataId[field] for quantum in self.quanta]
91class QuantumMock:
92 def __init__(self, dataId):
93 self.dataId = dataId
95 def __eq__(self, other):
96 return self.dataId == other.dataId
98 def __hash__(self):
99 # dict.__eq__ is order-insensitive
100 return hash(tuple(sorted(kv for kv in self.dataId.items())))
103class QuantumIterDataMock:
104 """Simple class to mock QuantumIterData."""
106 def __init__(self, index, taskDef, **dataId):
107 self.index = index
108 self.taskDef = taskDef
109 self.quantum = QuantumMock(dataId)
110 self.dependencies = set()
111 self.nodeId = NodeId(index, "DummyBuildString")
114class QuantumGraphMock:
115 """Mock for quantum graph."""
117 def __init__(self, qdata):
118 self._graph = nx.DiGraph()
119 previous = qdata[0]
120 for node in qdata[1:]:
121 self._graph.add_edge(previous, node)
122 previous = node
124 def __iter__(self):
125 yield from nx.topological_sort(self._graph)
127 def __len__(self):
128 return len(self._graph)
130 def findTaskDefByLabel(self, label):
131 for q in self:
132 if q.taskDef.label == label:
133 return q.taskDef
135 def getQuantaForTask(self, taskDef):
136 nodes = self.getNodesForTask(taskDef)
137 return {q.quantum for q in nodes}
139 def getNodesForTask(self, taskDef):
140 quanta = set()
141 for q in self:
142 if q.taskDef == taskDef:
143 quanta.add(q)
144 return quanta
146 @property
147 def graph(self):
148 return self._graph
150 def findCycle(self):
151 return []
153 def determineInputsToQuantumNode(self, node):
154 result = set()
155 for n in node.dependencies:
156 for otherNode in self:
157 if otherNode.index == n:
158 result.add(otherNode)
159 return result
162class TaskMockMP:
163 """Simple mock class for task supporting multiprocessing."""
165 canMultiprocess = True
167 def runQuantum(self):
168 _LOG.debug("TaskMockMP.runQuantum")
169 pass
172class TaskMockFail:
173 """Simple mock class for task which fails."""
175 canMultiprocess = True
177 def runQuantum(self):
178 _LOG.debug("TaskMockFail.runQuantum")
179 raise ValueError("expected failure")
182class TaskMockCrash:
183 """Simple mock class for task which fails."""
185 canMultiprocess = True
187 def runQuantum(self):
188 _LOG.debug("TaskMockCrash.runQuantum")
189 signal.raise_signal(signal.SIGILL)
192class TaskMockLongSleep:
193 """Simple mock class for task which "runs" for very long time."""
195 canMultiprocess = True
197 def runQuantum(self):
198 _LOG.debug("TaskMockLongSleep.runQuantum")
199 time.sleep(100.0)
202class TaskMockNoMP:
203 """Simple mock class for task not supporting multiprocessing."""
205 canMultiprocess = False
208class TaskDefMock:
209 """Simple mock class for task definition in a pipeline."""
211 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
212 self.taskName = taskName
213 self.config = config
214 self.taskClass = taskClass
215 self.label = label
217 def __str__(self):
218 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
221def _count_status(report, status):
222 """Count number of quanta witha a given status."""
223 return len([qrep for qrep in report.quantaReports if qrep.status is status])
226class MPGraphExecutorTestCase(unittest.TestCase):
227 """A test case for MPGraphExecutor class"""
229 def test_mpexec_nomp(self):
230 """Make simple graph and execute"""
232 taskDef = TaskDefMock()
233 qgraph = QuantumGraphMock(
234 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
235 )
237 # run in single-process mode
238 qexec = QuantumExecutorMock()
239 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
240 mpexec.execute(qgraph)
241 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
242 report = mpexec.getReport()
243 self.assertEqual(report.status, ExecutionStatus.SUCCESS)
244 self.assertIsNone(report.exitCode)
245 self.assertIsNone(report.exceptionInfo)
246 self.assertEqual(len(report.quantaReports), 3)
247 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports))
248 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports))
249 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
250 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports))
252 def test_mpexec_mp(self):
253 """Make simple graph and execute"""
255 taskDef = TaskDefMock()
256 qgraph = QuantumGraphMock(
257 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
258 )
260 methods = ["spawn"]
261 if sys.platform == "linux":
262 methods.append("fork")
263 methods.append("forkserver")
265 for method in methods:
266 with self.subTest(startMethod=method):
267 # Run in multi-process mode, the order of results is not
268 # defined.
269 qexec = QuantumExecutorMock(mp=True)
270 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method)
271 mpexec.execute(qgraph)
272 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
273 report = mpexec.getReport()
274 self.assertEqual(report.status, ExecutionStatus.SUCCESS)
275 self.assertIsNone(report.exitCode)
276 self.assertIsNone(report.exceptionInfo)
277 self.assertEqual(len(report.quantaReports), 3)
278 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports))
279 self.assertTrue(all(qrep.exitCode == 0 for qrep in report.quantaReports))
280 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
281 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports))
283 def test_mpexec_nompsupport(self):
284 """Try to run MP for task that has no MP support which should fail"""
286 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
287 qgraph = QuantumGraphMock(
288 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
289 )
291 # run in multi-process mode
292 qexec = QuantumExecutorMock()
293 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
294 with self.assertRaisesRegex(MPGraphExecutorError, "Task Task does not support multiprocessing"):
295 mpexec.execute(qgraph)
297 def test_mpexec_fixup(self):
298 """Make simple graph and execute, add dependencies by executing fixup
299 code.
300 """
302 taskDef = TaskDefMock()
304 for reverse in (False, True):
305 qgraph = QuantumGraphMock(
306 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)]
307 )
309 qexec = QuantumExecutorMock()
310 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
311 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, executionGraphFixup=fixup)
312 mpexec.execute(qgraph)
314 expected = [0, 1, 2]
315 if reverse:
316 expected = list(reversed(expected))
317 self.assertEqual(qexec.getDataIds("detector"), expected)
319 def test_mpexec_timeout(self):
320 """Fail due to timeout"""
322 taskDef = TaskDefMock()
323 taskDefSleep = TaskDefMock(taskClass=TaskMockLongSleep)
324 qgraph = QuantumGraphMock(
325 [
326 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
327 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
328 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
329 ]
330 )
332 # with failFast we'll get immediate MPTimeoutError
333 qexec = QuantumExecutorMock(mp=True)
334 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
335 with self.assertRaises(MPTimeoutError):
336 mpexec.execute(qgraph)
337 report = mpexec.getReport()
338 self.assertEqual(report.status, ExecutionStatus.TIMEOUT)
339 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError")
340 self.assertGreater(len(report.quantaReports), 0)
341 self.assertEqual(_count_status(report, ExecutionStatus.TIMEOUT), 1)
342 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports))
343 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
345 # with failFast=False exception happens after last task finishes
346 qexec = QuantumExecutorMock(mp=True)
347 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
348 with self.assertRaises(MPTimeoutError):
349 mpexec.execute(qgraph)
350 # We expect two tasks (0 and 2) to finish successfully and one task to
351 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
352 # finish on time, so expect more timeouts and issue a warning.
353 detectorIds = set(qexec.getDataIds("detector"))
354 self.assertLess(len(detectorIds), 3)
355 if detectorIds != {0, 2}:
356 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
357 report = mpexec.getReport()
358 self.assertEqual(report.status, ExecutionStatus.TIMEOUT)
359 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError")
360 self.assertGreater(len(report.quantaReports), 0)
361 self.assertGreater(_count_status(report, ExecutionStatus.TIMEOUT), 0)
362 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports))
363 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
365 def test_mpexec_failure(self):
366 """Failure in one task should not stop other tasks"""
368 taskDef = TaskDefMock()
369 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
370 qgraph = QuantumGraphMock(
371 [
372 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
373 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
374 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
375 ]
376 )
378 qexec = QuantumExecutorMock(mp=True)
379 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
380 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
381 mpexec.execute(qgraph)
382 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
383 report = mpexec.getReport()
384 self.assertEqual(report.status, ExecutionStatus.FAILURE)
385 self.assertEqual(
386 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
387 )
388 self.assertGreater(len(report.quantaReports), 0)
389 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
390 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
391 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
392 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
394 def test_mpexec_failure_dep(self):
395 """Failure in one task should skip dependents"""
397 taskDef = TaskDefMock()
398 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
399 qdata = [
400 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
401 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
402 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
403 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
404 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
405 ]
406 qdata[2].dependencies.add(1)
407 qdata[4].dependencies.add(3)
408 qdata[4].dependencies.add(2)
410 qgraph = QuantumGraphMock(qdata)
412 qexec = QuantumExecutorMock(mp=True)
413 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
414 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
415 mpexec.execute(qgraph)
416 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
417 report = mpexec.getReport()
418 self.assertEqual(report.status, ExecutionStatus.FAILURE)
419 self.assertEqual(
420 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
421 )
422 # Dependencies of failed tasks do not appear in quantaReports
423 self.assertGreater(len(report.quantaReports), 0)
424 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
425 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
426 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2)
427 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
428 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
430 def test_mpexec_failure_dep_nomp(self):
431 """Failure in one task should skip dependents, in-process version"""
433 taskDef = TaskDefMock()
434 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
435 qdata = [
436 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
437 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
438 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
439 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
440 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
441 ]
442 qdata[2].dependencies.add(1)
443 qdata[4].dependencies.add(3)
444 qdata[4].dependencies.add(2)
446 qgraph = QuantumGraphMock(qdata)
448 qexec = QuantumExecutorMock()
449 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
450 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
451 mpexec.execute(qgraph)
452 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
453 report = mpexec.getReport()
454 self.assertEqual(report.status, ExecutionStatus.FAILURE)
455 self.assertEqual(
456 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
457 )
458 # Dependencies of failed tasks do not appear in quantaReports
459 self.assertGreater(len(report.quantaReports), 0)
460 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
461 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
462 self.assertEqual(_count_status(report, ExecutionStatus.SKIPPED), 2)
463 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports))
464 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
466 def test_mpexec_failure_failfast(self):
467 """Fast fail stops quickly.
469 Timing delay of task #3 should be sufficient to process
470 failure and raise exception.
471 """
473 taskDef = TaskDefMock()
474 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
475 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
476 qdata = [
477 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
478 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
479 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
480 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
481 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
482 ]
483 qdata[1].dependencies.add(0)
484 qdata[2].dependencies.add(1)
485 qdata[4].dependencies.add(3)
486 qdata[4].dependencies.add(2)
488 qgraph = QuantumGraphMock(qdata)
490 qexec = QuantumExecutorMock(mp=True)
491 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
492 with self.assertRaisesRegex(MPGraphExecutorError, "failed, exit code=1"):
493 mpexec.execute(qgraph)
494 self.assertCountEqual(qexec.getDataIds("detector"), [0])
495 report = mpexec.getReport()
496 self.assertEqual(report.status, ExecutionStatus.FAILURE)
497 self.assertEqual(
498 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
499 )
500 # Dependencies of failed tasks do not appear in quantaReports
501 self.assertGreater(len(report.quantaReports), 0)
502 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
503 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports))
504 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports))
506 def test_mpexec_crash(self):
507 """Check task crash due to signal"""
509 taskDef = TaskDefMock()
510 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash)
511 qgraph = QuantumGraphMock(
512 [
513 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
514 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1),
515 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
516 ]
517 )
519 qexec = QuantumExecutorMock(mp=True)
520 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
521 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"):
522 mpexec.execute(qgraph)
523 report = mpexec.getReport()
524 self.assertEqual(report.status, ExecutionStatus.FAILURE)
525 self.assertEqual(
526 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
527 )
528 # Dependencies of failed tasks do not appear in quantaReports
529 self.assertGreater(len(report.quantaReports), 0)
530 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
531 self.assertEqual(_count_status(report, ExecutionStatus.SUCCESS), 2)
532 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports))
533 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
535 def test_mpexec_crash_failfast(self):
536 """Check task crash due to signal with --fail-fast"""
538 taskDef = TaskDefMock()
539 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash)
540 qgraph = QuantumGraphMock(
541 [
542 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
543 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1),
544 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
545 ]
546 )
548 qexec = QuantumExecutorMock(mp=True)
549 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
550 with self.assertRaisesRegex(MPGraphExecutorError, "failed, killed by signal 4 .Illegal instruction"):
551 mpexec.execute(qgraph)
552 report = mpexec.getReport()
553 self.assertEqual(report.status, ExecutionStatus.FAILURE)
554 self.assertEqual(
555 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError"
556 )
557 self.assertEqual(_count_status(report, ExecutionStatus.FAILURE), 1)
558 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports))
559 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports))
561 def test_mpexec_num_fd(self):
562 """Check that number of open files stays reasonable"""
564 taskDef = TaskDefMock()
565 qgraph = QuantumGraphMock(
566 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)]
567 )
569 this_proc = psutil.Process()
570 num_fds_0 = this_proc.num_fds()
572 # run in multi-process mode, the order of results is not defined
573 qexec = QuantumExecutorMock(mp=True)
574 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
575 mpexec.execute(qgraph)
577 num_fds_1 = this_proc.num_fds()
578 # They should be the same but allow small growth just in case.
579 # Without DM-26728 fix the difference would be equal to number of
580 # quanta (20).
581 self.assertLess(num_fds_1 - num_fds_0, 5)
584if __name__ == "__main__": 584 ↛ 585line 584 didn't jump to line 585, because the condition on line 584 was never true
585 unittest.main()