Coverage for tests/test_executors.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26import networkx as nx
27from multiprocessing import Manager
28import psutil
29import sys
30import time
31import unittest
32import warnings
34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
36from lsst.pipe.base import NodeId
39logging.basicConfig(level=logging.DEBUG)
41_LOG = logging.getLogger(__name__)
44class QuantumExecutorMock(QuantumExecutor):
45 """Mock class for QuantumExecutor
46 """
47 def __init__(self, mp=False):
48 self.quanta = []
49 if mp:
50 # in multiprocess mode use shared list
51 manager = Manager()
52 self.quanta = manager.list()
54 def execute(self, taskDef, quantum, butler):
55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
56 if taskDef.taskClass:
57 # only works for TaskMockMP class below
58 taskDef.taskClass().runQuantum()
59 self.quanta.append(quantum)
61 def getDataIds(self, field):
62 """Returns values for dataId field for each visited quanta"""
63 return [quantum.dataId[field] for quantum in self.quanta]
66class QuantumMock:
67 def __init__(self, dataId):
68 self.dataId = dataId
70 def __eq__(self, other):
71 return self.dataId == other.dataId
73 def __hash__(self):
74 # dict.__eq__ is order-insensitive
75 return hash(tuple(sorted(kv for kv in self.dataId.items())))
78class QuantumIterDataMock:
79 """Simple class to mock QuantumIterData.
80 """
81 def __init__(self, index, taskDef, **dataId):
82 self.index = index
83 self.taskDef = taskDef
84 self.quantum = QuantumMock(dataId)
85 self.dependencies = set()
86 self.nodeId = NodeId(index, "DummyBuildString")
89class QuantumGraphMock:
90 """Mock for quantum graph.
91 """
92 def __init__(self, qdata):
93 self._graph = nx.DiGraph()
94 previous = qdata[0]
95 for node in qdata[1:]:
96 self._graph.add_edge(previous, node)
97 previous = node
99 def __iter__(self):
100 yield from nx.topological_sort(self._graph)
102 def __len__(self):
103 return len(self._graph)
105 def findTaskDefByLabel(self, label):
106 for q in self:
107 if q.taskDef.label == label:
108 return q.taskDef
110 def getQuantaForTask(self, taskDef):
111 nodes = self.getNodesForTask(taskDef)
112 return {q.quantum for q in nodes}
114 def getNodesForTask(self, taskDef):
115 quanta = set()
116 for q in self:
117 if q.taskDef == taskDef:
118 quanta.add(q)
119 return quanta
121 @property
122 def graph(self):
123 return self._graph
125 def findCycle(self):
126 return []
128 def determineInputsToQuantumNode(self, node):
129 result = set()
130 for n in node.dependencies:
131 for otherNode in self:
132 if otherNode.index == n:
133 result.add(otherNode)
134 return result
137class TaskMockMP:
138 """Simple mock class for task supporting multiprocessing.
139 """
140 canMultiprocess = True
142 def runQuantum(self):
143 _LOG.debug("TaskMockMP.runQuantum")
144 pass
147class TaskMockFail:
148 """Simple mock class for task which fails.
149 """
150 canMultiprocess = True
152 def runQuantum(self):
153 _LOG.debug("TaskMockFail.runQuantum")
154 raise ValueError("expected failure")
157class TaskMockSleep:
158 """Simple mock class for task which "runs" for some time.
159 """
160 canMultiprocess = True
162 def runQuantum(self):
163 _LOG.debug("TaskMockSleep.runQuantum")
164 time.sleep(5.)
167class TaskMockLongSleep:
168 """Simple mock class for task which "runs" for very long time.
169 """
170 canMultiprocess = True
172 def runQuantum(self):
173 _LOG.debug("TaskMockLongSleep.runQuantum")
174 time.sleep(100.)
177class TaskMockNoMP:
178 """Simple mock class for task not supporting multiprocessing.
179 """
180 canMultiprocess = False
183class TaskDefMock:
184 """Simple mock class for task definition in a pipeline.
185 """
186 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
187 self.taskName = taskName
188 self.config = config
189 self.taskClass = taskClass
190 self.label = label
192 def __str__(self):
193 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
196class MPGraphExecutorTestCase(unittest.TestCase):
197 """A test case for MPGraphExecutor class
198 """
200 def test_mpexec_nomp(self):
201 """Make simple graph and execute"""
203 taskDef = TaskDefMock()
204 qgraph = QuantumGraphMock([
205 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
206 ])
208 # run in single-process mode
209 qexec = QuantumExecutorMock()
210 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
211 mpexec.execute(qgraph, butler=None)
212 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
214 def test_mpexec_mp(self):
215 """Make simple graph and execute"""
217 taskDef = TaskDefMock()
218 qgraph = QuantumGraphMock([
219 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
220 ])
222 methods = ["spawn"]
223 if sys.platform == "linux":
224 methods.append("fork")
225 methods.append("forkserver")
227 for method in methods:
228 with self.subTest(startMethod=method):
229 # run in multi-process mode, the order of results is not defined
230 qexec = QuantumExecutorMock(mp=True)
231 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method)
232 mpexec.execute(qgraph, butler=None)
233 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
235 def test_mpexec_nompsupport(self):
236 """Try to run MP for task that has no MP support which should fail
237 """
239 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
240 qgraph = QuantumGraphMock([
241 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
242 ])
244 # run in multi-process mode
245 qexec = QuantumExecutorMock()
246 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
247 with self.assertRaises(MPGraphExecutorError):
248 mpexec.execute(qgraph, butler=None)
250 def test_mpexec_fixup(self):
251 """Make simple graph and execute, add dependencies by executing fixup code
252 """
254 taskDef = TaskDefMock()
256 for reverse in (False, True):
257 qgraph = QuantumGraphMock([
258 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
259 ])
261 qexec = QuantumExecutorMock()
262 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
263 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec,
264 executionGraphFixup=fixup)
265 mpexec.execute(qgraph, butler=None)
267 expected = [0, 1, 2]
268 if reverse:
269 expected = list(reversed(expected))
270 self.assertEqual(qexec.getDataIds("detector"), expected)
272 def test_mpexec_timeout(self):
273 """Fail due to timeout"""
275 taskDef = TaskDefMock()
276 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
277 qgraph = QuantumGraphMock([
278 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
279 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
280 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
281 ])
283 # with failFast we'll get immediate MPTimeoutError
284 qexec = QuantumExecutorMock(mp=True)
285 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
286 with self.assertRaises(MPTimeoutError):
287 mpexec.execute(qgraph, butler=None)
289 # with failFast=False exception happens after last task finishes
290 qexec = QuantumExecutorMock(mp=True)
291 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
292 with self.assertRaises(MPTimeoutError):
293 mpexec.execute(qgraph, butler=None)
294 # We expect two tasks (0 and 2) to finish successfully and one task to
295 # timeout. Unfortunately on busy CPU there is no guarantee that tasks
296 # finish on time, so expect more timeouts and issue a warning.
297 detectorIds = set(qexec.getDataIds("detector"))
298 self.assertLess(len(detectorIds), 3)
299 if detectorIds != {0, 2}:
300 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")
302 def test_mpexec_failure(self):
303 """Failure in one task should not stop other tasks"""
305 taskDef = TaskDefMock()
306 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
307 qgraph = QuantumGraphMock([
308 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
309 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
310 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
311 ])
313 qexec = QuantumExecutorMock(mp=True)
314 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
315 with self.assertRaises(MPGraphExecutorError):
316 mpexec.execute(qgraph, butler=None)
317 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
319 def test_mpexec_failure_dep(self):
320 """Failure in one task should skip dependents"""
322 taskDef = TaskDefMock()
323 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
324 qdata = [
325 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
326 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
327 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
328 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
329 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
330 ]
331 qdata[2].dependencies.add(1)
332 qdata[4].dependencies.add(3)
333 qdata[4].dependencies.add(2)
335 qgraph = QuantumGraphMock(qdata)
337 qexec = QuantumExecutorMock(mp=True)
338 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
339 with self.assertRaises(MPGraphExecutorError):
340 mpexec.execute(qgraph, butler=None)
341 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
343 def test_mpexec_failure_failfast(self):
344 """Fast fail stops quickly.
346 Timing delay of task #3 should be sufficient to process
347 failure and raise exception.
348 """
350 taskDef = TaskDefMock()
351 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
352 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep)
353 qdata = [
354 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
355 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
356 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
357 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3),
358 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
359 ]
360 qdata[1].dependencies.add(0)
361 qdata[2].dependencies.add(1)
362 qdata[4].dependencies.add(3)
363 qdata[4].dependencies.add(2)
365 qgraph = QuantumGraphMock(qdata)
367 qexec = QuantumExecutorMock(mp=True)
368 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
369 with self.assertRaises(MPGraphExecutorError):
370 mpexec.execute(qgraph, butler=None)
371 self.assertCountEqual(qexec.getDataIds("detector"), [0])
373 def test_mpexec_num_fd(self):
374 """Check that number of open files stays reasonable
375 """
377 taskDef = TaskDefMock()
378 qgraph = QuantumGraphMock([
379 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)
380 ])
382 this_proc = psutil.Process()
383 num_fds_0 = this_proc.num_fds()
385 # run in multi-process mode, the order of results is not defined
386 qexec = QuantumExecutorMock(mp=True)
387 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
388 mpexec.execute(qgraph, butler=None)
390 num_fds_1 = this_proc.num_fds()
391 # They should be the same but allow small growth just in case.
392 # Without DM-26728 fix the difference would be equal to number of
393 # quanta (20).
394 self.assertLess(num_fds_1 - num_fds_0, 5)
397if __name__ == "__main__": 397 ↛ 398line 397 didn't jump to line 398, because the condition on line 397 was never true
398 unittest.main()