Coverage for tests/test_executors.py: 30%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26import networkx as nx
27from multiprocessing import Manager
28import psutil
29import sys
30import time
31import unittest
32import warnings
34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
36from lsst.pipe.base import NodeId
39logging.basicConfig(level=logging.DEBUG)
41_LOG = logging.getLogger(__name__)
44class QuantumExecutorMock(QuantumExecutor):
45 """Mock class for QuantumExecutor
46 """
47 def __init__(self, mp=False):
48 self.quanta = []
49 if mp:
50 # in multiprocess mode use shared list
51 manager = Manager()
52 self.quanta = manager.list()
54 def execute(self, taskDef, quantum, butler):
55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
56 if taskDef.taskClass:
57 # only works for TaskMockMP class below
58 taskDef.taskClass().runQuantum()
59 self.quanta.append(quantum)
61 def getDataIds(self, field):
62 """Returns values for dataId field for each visited quanta"""
63 return [quantum.dataId[field] for quantum in self.quanta]
66class QuantumMock:
67 def __init__(self, dataId):
68 self.dataId = dataId
70 def __eq__(self, other):
71 return self.dataId == other.dataId
73 def __hash__(self):
74 # dict.__eq__ is order-insensitive
75 return hash(tuple(sorted(kv for kv in self.dataId.items())))
78class QuantumIterDataMock:
79 """Simple class to mock QuantumIterData.
80 """
81 def __init__(self, index, taskDef, **dataId):
82 self.index = index
83 self.taskDef = taskDef
84 self.quantum = QuantumMock(dataId)
85 self.dependencies = set()
86 self.nodeId = NodeId(index, "DummyBuildString")
89class QuantumGraphMock:
90 """Mock for quantum graph.
91 """
92 def __init__(self, qdata):
93 self._graph = nx.DiGraph()
94 previous = qdata[0]
95 for node in qdata[1:]:
96 self._graph.add_edge(previous, node)
97 previous = node
99 def __iter__(self):
100 yield from nx.topological_sort(self._graph)
102 def __len__(self):
103 return len(self._graph)
105 def findTaskDefByLabel(self, label):
106 for q in self:
107 if q.taskDef.label == label:
108 return q.taskDef
110 def getQuantaForTask(self, taskDef):
111 nodes = self.getNodesForTask(taskDef)
112 return {q.quantum for q in nodes}
114 def getNodesForTask(self, taskDef):
115 quanta = set()
116 for q in self:
117 if q.taskDef == taskDef:
118 quanta.add(q)
119 return quanta
121 @property
122 def graph(self):
123 return self._graph
125 def findCycle(self):
126 return []
128 def determineInputsToQuantumNode(self, node):
129 result = set()
130 for n in node.dependencies:
131 for otherNode in self:
132 if otherNode.index == n:
133 result.add(otherNode)
134 return result
137class TaskMockMP:
138 """Simple mock class for task supporting multiprocessing.
139 """
140 canMultiprocess = True
142 def runQuantum(self):
143 _LOG.debug("TaskMockMP.runQuantum")
144 pass
147class TaskMockFail:
148 """Simple mock class for task which fails.
149 """
150 canMultiprocess = True
152 def runQuantum(self):
153 _LOG.debug("TaskMockFail.runQuantum")
154 raise ValueError("expected failure")
157class TaskMockSleep:
158 """Simple mock class for task which "runs" for some time.
159 """
160 canMultiprocess = True
162 def runQuantum(self):
163 _LOG.debug("TaskMockSleep.runQuantum")
164 time.sleep(5.)
167class TaskMockLongSleep:
168 """Simple mock class for task which "runs" for very long time.
169 """
170 canMultiprocess = True
172 def runQuantum(self):
173 _LOG.debug("TaskMockLongSleep.runQuantum")
174 time.sleep(100.)
177class TaskMockNoMP:
178 """Simple mock class for task not supporting multiprocessing.
179 """
180 canMultiprocess = False
183class TaskDefMock:
184 """Simple mock class for task definition in a pipeline.
185 """
186 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
187 self.taskName = taskName
188 self.config = config
189 self.taskClass = taskClass
190 self.label = label
192 def __str__(self):
193 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
196class MPGraphExecutorTestCase(unittest.TestCase):
197 """A test case for MPGraphExecutor class
198 """
200 def test_mpexec_nomp(self):
201 """Make simple graph and execute"""
203 taskDef = TaskDefMock()
204 qgraph = QuantumGraphMock([
205 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
206 ])
208 # run in single-process mode
209 qexec = QuantumExecutorMock()
210 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
211 mpexec.execute(qgraph, butler=None)
212 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
214 def test_mpexec_mp(self):
215 """Make simple graph and execute"""
217 taskDef = TaskDefMock()
218 qgraph = QuantumGraphMock([
219 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
220 ])
222 methods = ["spawn"]
223 if sys.platform == "linux":
224 methods.append("fork")
225 methods.append("forkserver")
227 for method in methods:
228 with self.subTest(startMethod=method):
229 # Run in multi-process mode, the order of results is not
230 # defined.
231 qexec = QuantumExecutorMock(mp=True)
232 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method)
233 mpexec.execute(qgraph, butler=None)
234 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
236 def test_mpexec_nompsupport(self):
237 """Try to run MP for task that has no MP support which should fail
238 """
240 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
241 qgraph = QuantumGraphMock([
242 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
243 ])
245 # run in multi-process mode
246 qexec = QuantumExecutorMock()
247 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
248 with self.assertRaises(MPGraphExecutorError):
249 mpexec.execute(qgraph, butler=None)
251 def test_mpexec_fixup(self):
252 """Make simple graph and execute, add dependencies by executing fixup
253 code.
254 """
256 taskDef = TaskDefMock()
258 for reverse in (False, True):
259 qgraph = QuantumGraphMock([
260 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
261 ])
263 qexec = QuantumExecutorMock()
264 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
265 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec,
266 executionGraphFixup=fixup)
267 mpexec.execute(qgraph, butler=None)
269 expected = [0, 1, 2]
270 if reverse:
271 expected = list(reversed(expected))
272 self.assertEqual(qexec.getDataIds("detector"), expected)
274 def test_mpexec_timeout(self):
275 """Fail due to timeout"""
277 taskDef = TaskDefMock()
278 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
279 qgraph = QuantumGraphMock([
280 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
281 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
282 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
283 ])
285 # with failFast we'll get immediate MPTimeoutError
286 qexec = QuantumExecutorMock(mp=True)
287 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
288 with self.assertRaises(MPTimeoutError):
289 mpexec.execute(qgraph, butler=None)
291 # with failFast=False exception happens after last task finishes
292 qexec = QuantumExecutorMock(mp=True)
293 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
294 with self.assertRaises(MPTimeoutError):
295 mpexec.execute(qgraph, butler=None)
296 # We expect two tasks (0 and 2) to finish successfully and one task to
297 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
298 # finish on time, so expect more timeouts and issue a warning.
299 detectorIds = set(qexec.getDataIds("detector"))
300 self.assertLess(len(detectorIds), 3)
301 if detectorIds != {0, 2}:
302 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
304 def test_mpexec_failure(self):
305 """Failure in one task should not stop other tasks"""
307 taskDef = TaskDefMock()
308 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
309 qgraph = QuantumGraphMock([
310 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
311 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
312 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
313 ])
315 qexec = QuantumExecutorMock(mp=True)
316 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
317 with self.assertRaises(MPGraphExecutorError):
318 mpexec.execute(qgraph, butler=None)
319 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
321 def test_mpexec_failure_dep(self):
322 """Failure in one task should skip dependents"""
324 taskDef = TaskDefMock()
325 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
326 qdata = [
327 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
328 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
329 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
330 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
331 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
332 ]
333 qdata[2].dependencies.add(1)
334 qdata[4].dependencies.add(3)
335 qdata[4].dependencies.add(2)
337 qgraph = QuantumGraphMock(qdata)
339 qexec = QuantumExecutorMock(mp=True)
340 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
341 with self.assertRaises(MPGraphExecutorError):
342 mpexec.execute(qgraph, butler=None)
343 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
345 def test_mpexec_failure_failfast(self):
346 """Fast fail stops quickly.
348 Timing delay of task #3 should be sufficient to process
349 failure and raise exception.
350 """
352 taskDef = TaskDefMock()
353 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
354 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
355 qdata = [
356 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
357 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
358 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
359 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
360 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
361 ]
362 qdata[1].dependencies.add(0)
363 qdata[2].dependencies.add(1)
364 qdata[4].dependencies.add(3)
365 qdata[4].dependencies.add(2)
367 qgraph = QuantumGraphMock(qdata)
369 qexec = QuantumExecutorMock(mp=True)
370 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
371 with self.assertRaises(MPGraphExecutorError):
372 mpexec.execute(qgraph, butler=None)
373 self.assertCountEqual(qexec.getDataIds("detector"), [0])
375 def test_mpexec_num_fd(self):
376 """Check that number of open files stays reasonable
377 """
379 taskDef = TaskDefMock()
380 qgraph = QuantumGraphMock([
381 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)
382 ])
384 this_proc = psutil.Process()
385 num_fds_0 = this_proc.num_fds()
387 # run in multi-process mode, the order of results is not defined
388 qexec = QuantumExecutorMock(mp=True)
389 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
390 mpexec.execute(qgraph, butler=None)
392 num_fds_1 = this_proc.num_fds()
393 # They should be the same but allow small growth just in case.
394 # Without DM-26728 fix the difference would be equal to number of
395 # quanta (20).
396 self.assertLess(num_fds_1 - num_fds_0, 5)
399if __name__ == "__main__": 399 ↛ 400line 399 didn't jump to line 400, because the condition on line 399 was never true
400 unittest.main()