Coverage for tests/test_executors.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26import networkx as nx
27from multiprocessing import Manager
28import psutil
29import time
30from types import SimpleNamespace
31import unittest
32import warnings
34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
36from lsst.pipe.base import NodeId
39logging.basicConfig(level=logging.DEBUG)
41_LOG = logging.getLogger(__name__)
44class QuantumExecutorMock(QuantumExecutor):
45 """Mock class for QuantumExecutor
46 """
47 def __init__(self, mp=False):
48 self.quanta = []
49 if mp:
50 # in multiprocess mode use shared list
51 manager = Manager()
52 self.quanta = manager.list()
54 def execute(self, taskDef, quantum, butler):
55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
56 if taskDef.taskClass:
57 # only works for TaskMockMP class below
58 taskDef.taskClass().runQuantum()
59 self.quanta.append(quantum)
61 def getDataIds(self, field):
62 """Returns values for dataId field for each visited quanta"""
63 return [quantum.dataId[field] for quantum in self.quanta]
66class QuantumIterDataMock:
67 """Simple class to mock QuantumIterData.
68 """
69 def __init__(self, index, taskDef, **dataId):
70 self.index = index
71 self.taskDef = taskDef
72 self.quantum = SimpleNamespace(dataId=dataId)
73 self.dependencies = set()
74 self.nodeId = NodeId(index, "DummyBuildString")
77class QuantumGraphMock:
78 """Mock for quantum graph.
79 """
80 def __init__(self, qdata):
81 self._graph = nx.DiGraph()
82 previous = qdata[0]
83 for node in qdata[1:]:
84 self._graph.add_edge(previous, node)
85 previous = node
87 def __iter__(self):
88 yield from nx.topological_sort(self._graph)
90 def findTaskDefByLabel(self, label):
91 for q in self:
92 if q.taskDef.label == label:
93 return q.taskDef
95 def quantaForTask(self, taskDef):
96 quanta = set()
97 for q in self:
98 if q.taskDef == taskDef:
99 quanta.add(q)
100 return quanta
102 @property
103 def graph(self):
104 return self._graph
106 def findCycle(self):
107 return []
109 def determineInputsToQuantumNode(self, node):
110 result = set()
111 for n in node.dependencies:
112 for otherNode in self:
113 if otherNode.index == n:
114 result.add(otherNode)
115 return result
118class TaskMockMP:
119 """Simple mock class for task supporting multiprocessing.
120 """
121 canMultiprocess = True
123 def runQuantum(self):
124 _LOG.debug("TaskMockMP.runQuantum")
125 pass
128class TaskMockFail:
129 """Simple mock class for task which fails.
130 """
131 canMultiprocess = True
133 def runQuantum(self):
134 _LOG.debug("TaskMockFail.runQuantum")
135 raise ValueError("expected failure")
138class TaskMockSleep:
139 """Simple mock class for task which fails.
140 """
141 canMultiprocess = True
143 def runQuantum(self):
144 _LOG.debug("TaskMockSleep.runQuantum")
145 time.sleep(5.)
148class TaskMockNoMP:
149 """Simple mock class for task not supporting multiprocessing.
150 """
151 canMultiprocess = False
154class TaskDefMock:
155 """Simple mock class for task definition in a pipeline.
156 """
157 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
158 self.taskName = taskName
159 self.config = config
160 self.taskClass = taskClass
161 self.label = label
163 def __str__(self):
164 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
167class MPGraphExecutorTestCase(unittest.TestCase):
168 """A test case for MPGraphExecutor class
169 """
171 def test_mpexec_nomp(self):
172 """Make simple graph and execute"""
174 taskDef = TaskDefMock()
175 qgraph = QuantumGraphMock([
176 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
177 ])
179 # run in single-process mode
180 qexec = QuantumExecutorMock()
181 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
182 mpexec.execute(qgraph, butler=None)
183 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
185 def test_mpexec_mp(self):
186 """Make simple graph and execute"""
188 taskDef = TaskDefMock()
189 qgraph = QuantumGraphMock([
190 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
191 ])
193 # run in multi-process mode, the order of results is not defined
194 qexec = QuantumExecutorMock(mp=True)
195 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
196 mpexec.execute(qgraph, butler=None)
197 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
199 def test_mpexec_nompsupport(self):
200 """Try to run MP for task that has no MP support which should fail
201 """
203 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
204 qgraph = QuantumGraphMock([
205 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
206 ])
208 # run in multi-process mode
209 qexec = QuantumExecutorMock()
210 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
211 with self.assertRaises(MPGraphExecutorError):
212 mpexec.execute(qgraph, butler=None)
214 def test_mpexec_fixup(self):
215 """Make simple graph and execute, add dependencies by executing fixup code
216 """
218 taskDef = TaskDefMock()
220 for reverse in (False, True):
221 qgraph = QuantumGraphMock([
222 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
223 ])
225 qexec = QuantumExecutorMock()
226 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
227 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec,
228 executionGraphFixup=fixup)
229 mpexec.execute(qgraph, butler=None)
231 expected = [0, 1, 2]
232 if reverse:
233 expected = list(reversed(expected))
234 self.assertEqual(qexec.getDataIds("detector"), expected)
236 def test_mpexec_timeout(self):
237 """Fail due to timeout"""
239 taskDef = TaskDefMock()
240 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
241 qgraph = QuantumGraphMock([
242 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
243 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
244 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
245 ])
247 # with failFast we'll get immediate MPTimeoutError
248 qexec = QuantumExecutorMock(mp=True)
249 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
250 with self.assertRaises(MPTimeoutError):
251 mpexec.execute(qgraph, butler=None)
253 # with failFast=False exception happens after last task finishes
254 qexec = QuantumExecutorMock(mp=True)
255 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
256 with self.assertRaises(MPTimeoutError):
257 mpexec.execute(qgraph, butler=None)
258 # We expect two tasks (0 and 2) to finish successfully and one task to
259 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
260 # finish on time, so expect more timeouts and issue a warning.
261 detectorIds = set(qexec.getDataIds("detector"))
262 self.assertLess(len(detectorIds), 3)
263 if detectorIds != {0, 2}:
264 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
266 def test_mpexec_failure(self):
267 """Failure in one task should not stop other tasks"""
269 taskDef = TaskDefMock()
270 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
271 qgraph = QuantumGraphMock([
272 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
273 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
274 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
275 ])
277 qexec = QuantumExecutorMock(mp=True)
278 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
279 with self.assertRaises(MPGraphExecutorError):
280 mpexec.execute(qgraph, butler=None)
281 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
283 def test_mpexec_failure_dep(self):
284 """Failure in one task should skip dependents"""
286 taskDef = TaskDefMock()
287 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
288 qdata = [
289 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
290 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
291 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
292 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
293 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
294 ]
295 qdata[2].dependencies.add(1)
296 qdata[4].dependencies.add(3)
297 qdata[4].dependencies.add(2)
299 qgraph = QuantumGraphMock(qdata)
301 qexec = QuantumExecutorMock(mp=True)
302 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
303 with self.assertRaises(MPGraphExecutorError):
304 mpexec.execute(qgraph, butler=None)
305 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
307 def test_mpexec_failure_failfast(self):
308 """Fast fail stops quickly.
310 Timing delay of task #3 should be sufficient to process
311 failure and raise exception.
312 """
314 taskDef = TaskDefMock()
315 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
316 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
317 qdata = [
318 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
319 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
320 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
321 QuantumIterDataMock(index=3, taskDef=taskDefSleep, detector=3),
322 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
323 ]
324 qdata[1].dependencies.add(0)
325 qdata[2].dependencies.add(1)
326 qdata[4].dependencies.add(3)
327 qdata[4].dependencies.add(2)
329 qgraph = QuantumGraphMock(qdata)
331 qexec = QuantumExecutorMock(mp=True)
332 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
333 with self.assertRaises(MPGraphExecutorError):
334 mpexec.execute(qgraph, butler=None)
335 self.assertCountEqual(qexec.getDataIds("detector"), [0])
337 def test_mpexec_num_fd(self):
338 """Check that number of open files stays reasonable
339 """
341 taskDef = TaskDefMock()
342 qgraph = QuantumGraphMock([
343 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)
344 ])
346 this_proc = psutil.Process()
347 num_fds_0 = this_proc.num_fds()
349 # run in multi-process mode, the order of results is not defined
350 qexec = QuantumExecutorMock(mp=True)
351 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
352 mpexec.execute(qgraph, butler=None)
354 num_fds_1 = this_proc.num_fds()
355 # They should be the same but allow small growth just in case.
356 # Without DM-26728 fix the difference would be equal to number of
357 # quanta (20).
358 self.assertLess(num_fds_1 - num_fds_0, 5)
361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true
362 unittest.main()