Coverage for tests/test_executors.py: 30%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26import networkx as nx
27from multiprocessing import Manager
28import psutil
29import sys
30import time
31import unittest
32import warnings
34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
36from lsst.pipe.base import NodeId
39logging.basicConfig(level=logging.DEBUG)
41_LOG = logging.getLogger(__name__)
44class QuantumExecutorMock(QuantumExecutor):
45 """Mock class for QuantumExecutor
46 """
47 def __init__(self, mp=False):
48 self.quanta = []
49 if mp:
50 # in multiprocess mode use shared list
51 manager = Manager()
52 self.quanta = manager.list()
54 def execute(self, taskDef, quantum, butler):
55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
56 if taskDef.taskClass:
57 # only works for TaskMockMP class below
58 taskDef.taskClass().runQuantum()
59 self.quanta.append(quantum)
60 return quantum
62 def getDataIds(self, field):
63 """Returns values for dataId field for each visited quanta"""
64 return [quantum.dataId[field] for quantum in self.quanta]
67class QuantumMock:
68 def __init__(self, dataId):
69 self.dataId = dataId
71 def __eq__(self, other):
72 return self.dataId == other.dataId
74 def __hash__(self):
75 # dict.__eq__ is order-insensitive
76 return hash(tuple(sorted(kv for kv in self.dataId.items())))
79class QuantumIterDataMock:
80 """Simple class to mock QuantumIterData.
81 """
82 def __init__(self, index, taskDef, **dataId):
83 self.index = index
84 self.taskDef = taskDef
85 self.quantum = QuantumMock(dataId)
86 self.dependencies = set()
87 self.nodeId = NodeId(index, "DummyBuildString")
90class QuantumGraphMock:
91 """Mock for quantum graph.
92 """
93 def __init__(self, qdata):
94 self._graph = nx.DiGraph()
95 previous = qdata[0]
96 for node in qdata[1:]:
97 self._graph.add_edge(previous, node)
98 previous = node
100 def __iter__(self):
101 yield from nx.topological_sort(self._graph)
103 def __len__(self):
104 return len(self._graph)
106 def findTaskDefByLabel(self, label):
107 for q in self:
108 if q.taskDef.label == label:
109 return q.taskDef
111 def getQuantaForTask(self, taskDef):
112 nodes = self.getNodesForTask(taskDef)
113 return {q.quantum for q in nodes}
115 def getNodesForTask(self, taskDef):
116 quanta = set()
117 for q in self:
118 if q.taskDef == taskDef:
119 quanta.add(q)
120 return quanta
122 @property
123 def graph(self):
124 return self._graph
126 def findCycle(self):
127 return []
129 def determineInputsToQuantumNode(self, node):
130 result = set()
131 for n in node.dependencies:
132 for otherNode in self:
133 if otherNode.index == n:
134 result.add(otherNode)
135 return result
138class TaskMockMP:
139 """Simple mock class for task supporting multiprocessing.
140 """
141 canMultiprocess = True
143 def runQuantum(self):
144 _LOG.debug("TaskMockMP.runQuantum")
145 pass
148class TaskMockFail:
149 """Simple mock class for task which fails.
150 """
151 canMultiprocess = True
153 def runQuantum(self):
154 _LOG.debug("TaskMockFail.runQuantum")
155 raise ValueError("expected failure")
158class TaskMockSleep:
159 """Simple mock class for task which "runs" for some time.
160 """
161 canMultiprocess = True
163 def runQuantum(self):
164 _LOG.debug("TaskMockSleep.runQuantum")
165 time.sleep(5.)
168class TaskMockLongSleep:
169 """Simple mock class for task which "runs" for very long time.
170 """
171 canMultiprocess = True
173 def runQuantum(self):
174 _LOG.debug("TaskMockLongSleep.runQuantum")
175 time.sleep(100.)
178class TaskMockNoMP:
179 """Simple mock class for task not supporting multiprocessing.
180 """
181 canMultiprocess = False
184class TaskDefMock:
185 """Simple mock class for task definition in a pipeline.
186 """
187 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
188 self.taskName = taskName
189 self.config = config
190 self.taskClass = taskClass
191 self.label = label
193 def __str__(self):
194 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
197class MPGraphExecutorTestCase(unittest.TestCase):
198 """A test case for MPGraphExecutor class
199 """
201 def test_mpexec_nomp(self):
202 """Make simple graph and execute"""
204 taskDef = TaskDefMock()
205 qgraph = QuantumGraphMock([
206 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
207 ])
209 # run in single-process mode
210 qexec = QuantumExecutorMock()
211 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
212 mpexec.execute(qgraph, butler=None)
213 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
215 def test_mpexec_mp(self):
216 """Make simple graph and execute"""
218 taskDef = TaskDefMock()
219 qgraph = QuantumGraphMock([
220 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
221 ])
223 methods = ["spawn"]
224 if sys.platform == "linux":
225 methods.append("fork")
226 methods.append("forkserver")
228 for method in methods:
229 with self.subTest(startMethod=method):
230 # Run in multi-process mode, the order of results is not
231 # defined.
232 qexec = QuantumExecutorMock(mp=True)
233 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method)
234 mpexec.execute(qgraph, butler=None)
235 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
237 def test_mpexec_nompsupport(self):
238 """Try to run MP for task that has no MP support which should fail
239 """
241 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
242 qgraph = QuantumGraphMock([
243 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
244 ])
246 # run in multi-process mode
247 qexec = QuantumExecutorMock()
248 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
249 with self.assertRaises(MPGraphExecutorError):
250 mpexec.execute(qgraph, butler=None)
252 def test_mpexec_fixup(self):
253 """Make simple graph and execute, add dependencies by executing fixup
254 code.
255 """
257 taskDef = TaskDefMock()
259 for reverse in (False, True):
260 qgraph = QuantumGraphMock([
261 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
262 ])
264 qexec = QuantumExecutorMock()
265 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
266 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec,
267 executionGraphFixup=fixup)
268 mpexec.execute(qgraph, butler=None)
270 expected = [0, 1, 2]
271 if reverse:
272 expected = list(reversed(expected))
273 self.assertEqual(qexec.getDataIds("detector"), expected)
275 def test_mpexec_timeout(self):
276 """Fail due to timeout"""
278 taskDef = TaskDefMock()
279 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
280 qgraph = QuantumGraphMock([
281 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
282 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
283 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
284 ])
286 # with failFast we'll get immediate MPTimeoutError
287 qexec = QuantumExecutorMock(mp=True)
288 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
289 with self.assertRaises(MPTimeoutError):
290 mpexec.execute(qgraph, butler=None)
292 # with failFast=False exception happens after last task finishes
293 qexec = QuantumExecutorMock(mp=True)
294 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
295 with self.assertRaises(MPTimeoutError):
296 mpexec.execute(qgraph, butler=None)
297 # We expect two tasks (0 and 2) to finish successfully and one task to
298 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
299 # finish on time, so expect more timeouts and issue a warning.
300 detectorIds = set(qexec.getDataIds("detector"))
301 self.assertLess(len(detectorIds), 3)
302 if detectorIds != {0, 2}:
303 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
305 def test_mpexec_failure(self):
306 """Failure in one task should not stop other tasks"""
308 taskDef = TaskDefMock()
309 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
310 qgraph = QuantumGraphMock([
311 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
312 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
313 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
314 ])
316 qexec = QuantumExecutorMock(mp=True)
317 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
318 with self.assertRaises(MPGraphExecutorError):
319 mpexec.execute(qgraph, butler=None)
320 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
322 def test_mpexec_failure_dep(self):
323 """Failure in one task should skip dependents"""
325 taskDef = TaskDefMock()
326 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
327 qdata = [
328 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
329 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
330 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
331 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
332 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
333 ]
334 qdata[2].dependencies.add(1)
335 qdata[4].dependencies.add(3)
336 qdata[4].dependencies.add(2)
338 qgraph = QuantumGraphMock(qdata)
340 qexec = QuantumExecutorMock(mp=True)
341 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
342 with self.assertRaises(MPGraphExecutorError):
343 mpexec.execute(qgraph, butler=None)
344 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
346 def test_mpexec_failure_failfast(self):
347 """Fast fail stops quickly.
349 Timing delay of task #3 should be sufficient to process
350 failure and raise exception.
351 """
353 taskDef = TaskDefMock()
354 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
355 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
356 qdata = [
357 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
358 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
359 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
360 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
361 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
362 ]
363 qdata[1].dependencies.add(0)
364 qdata[2].dependencies.add(1)
365 qdata[4].dependencies.add(3)
366 qdata[4].dependencies.add(2)
368 qgraph = QuantumGraphMock(qdata)
370 qexec = QuantumExecutorMock(mp=True)
371 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
372 with self.assertRaises(MPGraphExecutorError):
373 mpexec.execute(qgraph, butler=None)
374 self.assertCountEqual(qexec.getDataIds("detector"), [0])
376 def test_mpexec_num_fd(self):
377 """Check that number of open files stays reasonable
378 """
380 taskDef = TaskDefMock()
381 qgraph = QuantumGraphMock([
382 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)
383 ])
385 this_proc = psutil.Process()
386 num_fds_0 = this_proc.num_fds()
388 # run in multi-process mode, the order of results is not defined
389 qexec = QuantumExecutorMock(mp=True)
390 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
391 mpexec.execute(qgraph, butler=None)
393 num_fds_1 = this_proc.num_fds()
394 # They should be the same but allow small growth just in case.
395 # Without DM-26728 fix the difference would be equal to number of
396 # quanta (20).
397 self.assertLess(num_fds_1 - num_fds_0, 5)
400if __name__ == "__main__": 400 ↛ 401line 400 didn't jump to line 401, because the condition on line 400 was never true
401 unittest.main()