Coverage for tests/test_executors.py : 26%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import logging
26from multiprocessing import Manager
27import psutil
28import time
29from types import SimpleNamespace
30import unittest
32from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
33from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
36logging.basicConfig(level=logging.DEBUG)
38_LOG = logging.getLogger(__name__)
41class QuantumExecutorMock(QuantumExecutor):
42 """Mock class for QuantumExecutor
43 """
44 def __init__(self, mp=False):
45 self.quanta = []
46 if mp:
47 # in multiprocess mode use shared list
48 manager = Manager()
49 self.quanta = manager.list()
51 def execute(self, taskDef, quantum, butler):
52 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId)
53 if taskDef.taskClass:
54 # only works for TaskMockMP class below
55 taskDef.taskClass().runQuantum()
56 self.quanta.append(quantum)
58 def getDataIds(self, field):
59 """Returns values for dataId field for each visited quanta"""
60 return [quantum.dataId[field] for quantum in self.quanta]
63class QuantumIterDataMock:
64 """Simple class to mock QuantumIterData.
65 """
66 def __init__(self, index, taskDef, **dataId):
67 self.index = index
68 self.taskDef = taskDef
69 self.quantum = SimpleNamespace(dataId=dataId)
70 self.dependencies = set()
73class QuantumGraphMock:
74 """Mock for quantum graph.
75 """
76 def __init__(self, qdata):
77 self.qdata = qdata
79 def traverse(self):
80 return self.qdata
83class TaskMockMP:
84 """Simple mock class for task supporting multiprocessing.
85 """
86 canMultiprocess = True
88 def runQuantum(self):
89 _LOG.debug("TaskMockMP.runQuantum")
90 pass
93class TaskMockFail:
94 """Simple mock class for task which fails.
95 """
96 canMultiprocess = True
98 def runQuantum(self):
99 _LOG.debug("TaskMockFail.runQuantum")
100 raise ValueError("expected failure")
103class TaskMockSleep:
104 """Simple mock class for task which fails.
105 """
106 canMultiprocess = True
108 def runQuantum(self):
109 _LOG.debug("TaskMockSleep.runQuantum")
110 time.sleep(3.)
113class TaskMockNoMP:
114 """Simple mock class for task not supporting multiprocessing.
115 """
116 canMultiprocess = False
119class TaskDefMock:
120 """Simple mock class for task definition in a pipeline.
121 """
122 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"):
123 self.taskName = taskName
124 self.config = config
125 self.taskClass = taskClass
126 self.label = label
128 def __str__(self):
129 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})"
132class MPGraphExecutorTestCase(unittest.TestCase):
133 """A test case for MPGraphExecutor class
134 """
136 def test_mpexec_nomp(self):
137 """Make simple graph and execute"""
139 taskDef = TaskDefMock()
140 qgraph = QuantumGraphMock([
141 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
142 ])
144 # run in single-process mode
145 qexec = QuantumExecutorMock()
146 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec)
147 mpexec.execute(qgraph, butler=None)
148 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2])
150 def test_mpexec_mp(self):
151 """Make simple graph and execute"""
153 taskDef = TaskDefMock()
154 qgraph = QuantumGraphMock([
155 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
156 ])
158 # run in multi-process mode, the order of results is not defined
159 qexec = QuantumExecutorMock(mp=True)
160 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
161 mpexec.execute(qgraph, butler=None)
162 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2])
164 def test_mpexec_nompsupport(self):
165 """Try to run MP for task that has no MP support which should fail
166 """
168 taskDef = TaskDefMock(taskClass=TaskMockNoMP)
169 qgraph = QuantumGraphMock([
170 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
171 ])
173 # run in multi-process mode
174 qexec = QuantumExecutorMock()
175 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
176 with self.assertRaises(MPGraphExecutorError):
177 mpexec.execute(qgraph, butler=None)
179 def test_mpexec_fixup(self):
180 """Make simple graph and execute, add dependencies by executing fixup code
181 """
183 taskDef = TaskDefMock()
185 for reverse in (False, True):
187 qgraph = QuantumGraphMock([
188 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)
189 ])
191 qexec = QuantumExecutorMock()
192 fixup = ExecFixupDataId("task1", "detector", reverse=reverse)
193 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec,
194 executionGraphFixup=fixup)
195 mpexec.execute(qgraph, butler=None)
197 expected = [0, 1, 2]
198 if reverse:
199 expected = list(reversed(expected))
200 self.assertEqual(qexec.getDataIds("detector"), expected)
202 def test_mpexec_timeout(self):
203 """Fail due to timeout"""
205 taskDef = TaskDefMock()
206 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
207 qgraph = QuantumGraphMock([
208 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
209 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1),
210 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
211 ])
213 # with failFast we'll get immediate MPTimeoutError
214 qexec = QuantumExecutorMock(mp=True)
215 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True)
216 with self.assertRaises(MPTimeoutError):
217 mpexec.execute(qgraph, butler=None)
219 # with failFast=False exception happens after last task finishes
220 qexec = QuantumExecutorMock(mp=True)
221 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=False)
222 with self.assertRaises(MPTimeoutError):
223 mpexec.execute(qgraph, butler=None)
224 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
226 def test_mpexec_failure(self):
227 """Failure in one task should not stop other tasks"""
229 taskDef = TaskDefMock()
230 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
231 qgraph = QuantumGraphMock([
232 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
233 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
234 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
235 ])
237 qexec = QuantumExecutorMock(mp=True)
238 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
239 with self.assertRaises(MPGraphExecutorError):
240 mpexec.execute(qgraph, butler=None)
241 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
243 def test_mpexec_failure_dep(self):
244 """Failure in one task should skip dependents"""
246 taskDef = TaskDefMock()
247 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
248 qdata = [
249 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
250 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
251 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
252 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3),
253 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
254 ]
255 qdata[2].dependencies.add(1)
256 qdata[4].dependencies.add(3)
257 qdata[4].dependencies.add(2)
259 qgraph = QuantumGraphMock(qdata)
261 qexec = QuantumExecutorMock(mp=True)
262 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
263 with self.assertRaises(MPGraphExecutorError):
264 mpexec.execute(qgraph, butler=None)
265 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3])
267 def test_mpexec_failure_failfast(self):
268 """Fast fail stops quickly.
270 Timing delay of task #3 should be sufficient to process
271 failure and raise exception.
272 """
274 taskDef = TaskDefMock()
275 taskDefFail = TaskDefMock(taskClass=TaskMockFail)
276 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep)
277 qdata = [
278 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0),
279 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1),
280 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2),
281 QuantumIterDataMock(index=3, taskDef=taskDefSleep, detector=3),
282 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4),
283 ]
284 qdata[1].dependencies.add(0)
285 qdata[2].dependencies.add(1)
286 qdata[4].dependencies.add(3)
287 qdata[4].dependencies.add(2)
289 qgraph = QuantumGraphMock(qdata)
291 qexec = QuantumExecutorMock(mp=True)
292 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True)
293 with self.assertRaises(MPGraphExecutorError):
294 mpexec.execute(qgraph, butler=None)
295 self.assertCountEqual(qexec.getDataIds("detector"), [0])
297 def test_mpexec_num_fd(self):
298 """Check that number of open files stays reasonable
299 """
301 taskDef = TaskDefMock()
302 qgraph = QuantumGraphMock([
303 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)
304 ])
306 this_proc = psutil.Process()
307 num_fds_0 = this_proc.num_fds()
309 # run in multi-process mode, the order of results is not defined
310 qexec = QuantumExecutorMock(mp=True)
311 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec)
312 mpexec.execute(qgraph, butler=None)
314 num_fds_1 = this_proc.num_fds()
315 # They should be the same but allow small growth just in case.
316 # Without DM-26728 fix the difference would be equal to number of
317 # quanta (20).
318 self.assertLess(num_fds_1 - num_fds_0, 5)
321if __name__ == "__main__": 321 ↛ 322line 321 didn't jump to line 322, because the condition on line 321 was never true
322 unittest.main()