Coverage for tests/test_executors.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26import networkx as nx
27from multiprocessing import Manager
28import psutil
29import time
30from types import SimpleNamespace
31import unittest
32import warnings
34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
36from lsst.pipe.base import NodeId
39logging.basicConfig(level=logging.DEBUG)
41_LOG = logging.getLogger(__name__)
44class QuantumExecutorMock(QuantumExecutor):
45 """Mock class for QuantumExecutor
46 """
47 def __init__(self, mp=False):
48 self.quanta = []
49 if mp:
50 # in multiprocess mode use shared list
51 manager = Manager()
52 self.quanta = manager.list()
54 def execute(self, taskDef, quantum, butler):
55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
56 if taskDef.taskClass:
57 # only works for TaskMockMP class below
58 taskDef.taskClass().runQuantum()
59 self.quanta.append(quantum)
61 def getDataIds(self, field):
62 """Returns values for dataId field for each visited quanta"""
63 return [quantum.dataId[field] for quantum in self.quanta]
66class QuantumIterDataMock:
67 """Simple class to mock QuantumIterData.
68 """
69 def __init__(self, index, taskDef, **dataId):
70 self.index = index
71 self.taskDef = taskDef
72 self.quantum = SimpleNamespace(dataId=dataId)
73 self.dependencies = set()
74 self.nodeId = NodeId(index, "DummyBuildString")
77class QuantumGraphMock:
78 """Mock for quantum graph.
79 """
80 def __init__(self, qdata):
81 self._graph = nx.DiGraph()
82 previous = qdata[0]
83 for node in qdata[1:]:
84 self._graph.add_edge(previous, node)
85 previous = node
87 def __iter__(self):
88 yield from nx.topological_sort(self._graph)
90 def __len__(self):
91 return len(self._graph)
93 def findTaskDefByLabel(self, label):
94 for q in self:
95 if q.taskDef.label == label:
96 return q.taskDef
98 def quantaForTask(self, taskDef):
99 quanta = set()
100 for q in self:
101 if q.taskDef == taskDef:
102 quanta.add(q)
103 return quanta
105 @property
106 def graph(self):
107 return self._graph
109 def findCycle(self):
110 return []
112 def determineInputsToQuantumNode(self, node):
113 result = set()
114 for n in node.dependencies:
115 for otherNode in self:
116 if otherNode.index == n:
117 result.add(otherNode)
118 return result
121class TaskMockMP:
122 """Simple mock class for task supporting multiprocessing.
123 """
124 canMultiprocess = True
126 def runQuantum(self):
127 _LOG.debug("TaskMockMP.runQuantum")
128 pass
131class TaskMockFail:
132 """Simple mock class for task which fails.
133 """
134 canMultiprocess = True
136 def runQuantum(self):
137 _LOG.debug("TaskMockFail.runQuantum")
138 raise ValueError("expected failure")
141class TaskMockSleep:
142 """Simple mock class for task which "runs" for some time.
143 """
144 canMultiprocess = True
146 def runQuantum(self):
147 _LOG.debug("TaskMockSleep.runQuantum")
148 time.sleep(5.)
151class TaskMockLongSleep:
152 """Simple mock class for task which "runs" for very long time.
153 """
154 canMultiprocess = True
156 def runQuantum(self):
157 _LOG.debug("TaskMockLongSleep.runQuantum")
158 time.sleep(100.)
161class TaskMockNoMP:
162 """Simple mock class for task not supporting multiprocessing.
163 """
164 canMultiprocess = False
167class TaskDefMock:
168 """Simple mock class for task definition in a pipeline.
169 """
170 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
171 self.taskName = taskName
172 self.config = config
173 self.taskClass = taskClass
174 self.label = label
176 def __str__(self):
177 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
180class MPGraphExecutorTestCase(unittest.TestCase):
181 """A test case for MPGraphExecutor class
182 """
184 def test_mpexec_nomp(self):
185 """Make simple graph and execute"""
187 taskDef = TaskDefMock()
188 qgraph = QuantumGraphMock([
189 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
190 ])
192 # run in single-process mode
193 qexec = QuantumExecutorMock()
194 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
195 mpexec.execute(qgraph, butler=None)
196 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
198 def test_mpexec_mp(self):
199 """Make simple graph and execute"""
201 taskDef = TaskDefMock()
202 qgraph = QuantumGraphMock([
203 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
204 ])
206 # run in multi-process mode, the order of results is not defined
207 qexec = QuantumExecutorMock(mp=True)
208 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
209 mpexec.execute(qgraph, butler=None)
210 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
212 def test_mpexec_nompsupport(self):
213 """Try to run MP for task that has no MP support which should fail
214 """
216 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
217 qgraph = QuantumGraphMock([
218 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
219 ])
221 # run in multi-process mode
222 qexec = QuantumExecutorMock()
223 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
224 with self.assertRaises(MPGraphExecutorError):
225 mpexec.execute(qgraph, butler=None)
227 def test_mpexec_fixup(self):
228 """Make simple graph and execute, add dependencies by executing fixup code
229 """
231 taskDef = TaskDefMock()
233 for reverse in (False, True):
234 qgraph = QuantumGraphMock([
235 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
236 ])
238 qexec = QuantumExecutorMock()
239 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
240 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec,
241 executionGraphFixup=fixup)
242 mpexec.execute(qgraph, butler=None)
244 expected = [0, 1, 2]
245 if reverse:
246 expected = list(reversed(expected))
247 self.assertEqual(qexec.getDataIds("detector"), expected)
249 def test_mpexec_timeout(self):
250 """Fail due to timeout"""
252 taskDef = TaskDefMock()
253 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
254 qgraph = QuantumGraphMock([
255 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
256 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
257 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
258 ])
260 # with failFast we'll get immediate MPTimeoutError
261 qexec = QuantumExecutorMock(mp=True)
262 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
263 with self.assertRaises(MPTimeoutError):
264 mpexec.execute(qgraph, butler=None)
266 # with failFast=False exception happens after last task finishes
267 qexec = QuantumExecutorMock(mp=True)
268 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
269 with self.assertRaises(MPTimeoutError):
270 mpexec.execute(qgraph, butler=None)
271 # We expect two tasks (0 and 2) to finish successfully and one task to
272 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
273 # finish on time, so expect more timeouts and issue a warning.
274 detectorIds = set(qexec.getDataIds("detector"))
275 self.assertLess(len(detectorIds), 3)
276 if detectorIds != {0, 2}:
277 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
279 def test_mpexec_failure(self):
280 """Failure in one task should not stop other tasks"""
282 taskDef = TaskDefMock()
283 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
284 qgraph = QuantumGraphMock([
285 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
286 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
287 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
288 ])
290 qexec = QuantumExecutorMock(mp=True)
291 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
292 with self.assertRaises(MPGraphExecutorError):
293 mpexec.execute(qgraph, butler=None)
294 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
296 def test_mpexec_failure_dep(self):
297 """Failure in one task should skip dependents"""
299 taskDef = TaskDefMock()
300 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
301 qdata = [
302 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
303 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
304 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
305 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
306 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
307 ]
308 qdata[2].dependencies.add(1)
309 qdata[4].dependencies.add(3)
310 qdata[4].dependencies.add(2)
312 qgraph = QuantumGraphMock(qdata)
314 qexec = QuantumExecutorMock(mp=True)
315 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
316 with self.assertRaises(MPGraphExecutorError):
317 mpexec.execute(qgraph, butler=None)
318 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
320 def test_mpexec_failure_failfast(self):
321 """Fast fail stops quickly.
323 Timing delay of task #3 should be sufficient to process
324 failure and raise exception.
325 """
327 taskDef = TaskDefMock()
328 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
329 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
330 qdata = [
331 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
332 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
333 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
334 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
335 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
336 ]
337 qdata[1].dependencies.add(0)
338 qdata[2].dependencies.add(1)
339 qdata[4].dependencies.add(3)
340 qdata[4].dependencies.add(2)
342 qgraph = QuantumGraphMock(qdata)
344 qexec = QuantumExecutorMock(mp=True)
345 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
346 with self.assertRaises(MPGraphExecutorError):
347 mpexec.execute(qgraph, butler=None)
348 self.assertCountEqual(qexec.getDataIds("detector"), [0])
350 def test_mpexec_num_fd(self):
351 """Check that number of open files stays reasonable
352 """
354 taskDef = TaskDefMock()
355 qgraph = QuantumGraphMock([
356 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)
357 ])
359 this_proc = psutil.Process()
360 num_fds_0 = this_proc.num_fds()
362 # run in multi-process mode, the order of results is not defined
363 qexec = QuantumExecutorMock(mp=True)
364 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
365 mpexec.execute(qgraph, butler=None)
367 num_fds_1 = this_proc.num_fds()
368 # They should be the same but allow small growth just in case.
369 # Without DM-26728 fix the difference would be equal to number of
370 # quanta (20).
371 self.assertLess(num_fds_1 - num_fds_0, 5)
374if __name__ == "__main__": 374 ↛ 375line 374 didn't jump to line 375, because the condition on line 374 was never true
375 unittest.main()