Coverage for tests/test_executors.py: 25%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

213 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import networkx as nx 

27from multiprocessing import Manager 

28import psutil 

29import sys 

30import time 

31import unittest 

32import warnings 

33 

34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor 

35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

36from lsst.pipe.base import NodeId 

37 

38 

39logging.basicConfig(level=logging.DEBUG) 

40 

41_LOG = logging.getLogger(__name__) 

42 

43 

44class QuantumExecutorMock(QuantumExecutor): 

45 """Mock class for QuantumExecutor 

46 """ 

47 def __init__(self, mp=False): 

48 self.quanta = [] 

49 if mp: 

50 # in multiprocess mode use shared list 

51 manager = Manager() 

52 self.quanta = manager.list() 

53 

54 def execute(self, taskDef, quantum, butler): 

55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

56 if taskDef.taskClass: 

57 # only works for TaskMockMP class below 

58 taskDef.taskClass().runQuantum() 

59 self.quanta.append(quantum) 

60 return quantum 

61 

62 def getDataIds(self, field): 

63 """Returns values for dataId field for each visited quanta""" 

64 return [quantum.dataId[field] for quantum in self.quanta] 

65 

66 

67class QuantumMock: 

68 def __init__(self, dataId): 

69 self.dataId = dataId 

70 

71 def __eq__(self, other): 

72 return self.dataId == other.dataId 

73 

74 def __hash__(self): 

75 # dict.__eq__ is order-insensitive 

76 return hash(tuple(sorted(kv for kv in self.dataId.items()))) 

77 

78 

79class QuantumIterDataMock: 

80 """Simple class to mock QuantumIterData. 

81 """ 

82 def __init__(self, index, taskDef, **dataId): 

83 self.index = index 

84 self.taskDef = taskDef 

85 self.quantum = QuantumMock(dataId) 

86 self.dependencies = set() 

87 self.nodeId = NodeId(index, "DummyBuildString") 

88 

89 

90class QuantumGraphMock: 

91 """Mock for quantum graph. 

92 """ 

93 def __init__(self, qdata): 

94 self._graph = nx.DiGraph() 

95 previous = qdata[0] 

96 for node in qdata[1:]: 

97 self._graph.add_edge(previous, node) 

98 previous = node 

99 

100 def __iter__(self): 

101 yield from nx.topological_sort(self._graph) 

102 

103 def __len__(self): 

104 return len(self._graph) 

105 

106 def findTaskDefByLabel(self, label): 

107 for q in self: 

108 if q.taskDef.label == label: 

109 return q.taskDef 

110 

111 def getQuantaForTask(self, taskDef): 

112 nodes = self.getNodesForTask(taskDef) 

113 return {q.quantum for q in nodes} 

114 

115 def getNodesForTask(self, taskDef): 

116 quanta = set() 

117 for q in self: 

118 if q.taskDef == taskDef: 

119 quanta.add(q) 

120 return quanta 

121 

122 @property 

123 def graph(self): 

124 return self._graph 

125 

126 def findCycle(self): 

127 return [] 

128 

129 def determineInputsToQuantumNode(self, node): 

130 result = set() 

131 for n in node.dependencies: 

132 for otherNode in self: 

133 if otherNode.index == n: 

134 result.add(otherNode) 

135 return result 

136 

137 

138class TaskMockMP: 

139 """Simple mock class for task supporting multiprocessing. 

140 """ 

141 canMultiprocess = True 

142 

143 def runQuantum(self): 

144 _LOG.debug("TaskMockMP.runQuantum") 

145 pass 

146 

147 

148class TaskMockFail: 

149 """Simple mock class for task which fails. 

150 """ 

151 canMultiprocess = True 

152 

153 def runQuantum(self): 

154 _LOG.debug("TaskMockFail.runQuantum") 

155 raise ValueError("expected failure") 

156 

157 

158class TaskMockSleep: 

159 """Simple mock class for task which "runs" for some time. 

160 """ 

161 canMultiprocess = True 

162 

163 def runQuantum(self): 

164 _LOG.debug("TaskMockSleep.runQuantum") 

165 time.sleep(5.) 

166 

167 

168class TaskMockLongSleep: 

169 """Simple mock class for task which "runs" for very long time. 

170 """ 

171 canMultiprocess = True 

172 

173 def runQuantum(self): 

174 _LOG.debug("TaskMockLongSleep.runQuantum") 

175 time.sleep(100.) 

176 

177 

178class TaskMockNoMP: 

179 """Simple mock class for task not supporting multiprocessing. 

180 """ 

181 canMultiprocess = False 

182 

183 

184class TaskDefMock: 

185 """Simple mock class for task definition in a pipeline. 

186 """ 

187 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

188 self.taskName = taskName 

189 self.config = config 

190 self.taskClass = taskClass 

191 self.label = label 

192 

193 def __str__(self): 

194 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

195 

196 

197class MPGraphExecutorTestCase(unittest.TestCase): 

198 """A test case for MPGraphExecutor class 

199 """ 

200 

201 def test_mpexec_nomp(self): 

202 """Make simple graph and execute""" 

203 

204 taskDef = TaskDefMock() 

205 qgraph = QuantumGraphMock([ 

206 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

207 ]) 

208 

209 # run in single-process mode 

210 qexec = QuantumExecutorMock() 

211 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

212 mpexec.execute(qgraph, butler=None) 

213 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

214 

215 def test_mpexec_mp(self): 

216 """Make simple graph and execute""" 

217 

218 taskDef = TaskDefMock() 

219 qgraph = QuantumGraphMock([ 

220 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

221 ]) 

222 

223 methods = ["spawn"] 

224 if sys.platform == "linux": 

225 methods.append("fork") 

226 methods.append("forkserver") 

227 

228 for method in methods: 

229 with self.subTest(startMethod=method): 

230 # Run in multi-process mode, the order of results is not 

231 # defined. 

232 qexec = QuantumExecutorMock(mp=True) 

233 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method) 

234 mpexec.execute(qgraph, butler=None) 

235 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

236 

237 def test_mpexec_nompsupport(self): 

238 """Try to run MP for task that has no MP support which should fail 

239 """ 

240 

241 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

242 qgraph = QuantumGraphMock([ 

243 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

244 ]) 

245 

246 # run in multi-process mode 

247 qexec = QuantumExecutorMock() 

248 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

249 with self.assertRaises(MPGraphExecutorError): 

250 mpexec.execute(qgraph, butler=None) 

251 

252 def test_mpexec_fixup(self): 

253 """Make simple graph and execute, add dependencies by executing fixup 

254 code. 

255 """ 

256 

257 taskDef = TaskDefMock() 

258 

259 for reverse in (False, True): 

260 qgraph = QuantumGraphMock([ 

261 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

262 ]) 

263 

264 qexec = QuantumExecutorMock() 

265 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

266 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, 

267 executionGraphFixup=fixup) 

268 mpexec.execute(qgraph, butler=None) 

269 

270 expected = [0, 1, 2] 

271 if reverse: 

272 expected = list(reversed(expected)) 

273 self.assertEqual(qexec.getDataIds("detector"), expected) 

274 

275 def test_mpexec_timeout(self): 

276 """Fail due to timeout""" 

277 

278 taskDef = TaskDefMock() 

279 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

280 qgraph = QuantumGraphMock([ 

281 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

282 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

283 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

284 ]) 

285 

286 # with failFast we'll get immediate MPTimeoutError 

287 qexec = QuantumExecutorMock(mp=True) 

288 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

289 with self.assertRaises(MPTimeoutError): 

290 mpexec.execute(qgraph, butler=None) 

291 

292 # with failFast=False exception happens after last task finishes 

293 qexec = QuantumExecutorMock(mp=True) 

294 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

295 with self.assertRaises(MPTimeoutError): 

296 mpexec.execute(qgraph, butler=None) 

297 # We expect two tasks (0 and 2) to finish successfully and one task to 

298 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

299 # finish on time, so expect more timeouts and issue a warning. 

300 detectorIds = set(qexec.getDataIds("detector")) 

301 self.assertLess(len(detectorIds), 3) 

302 if detectorIds != {0, 2}: 

303 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

304 

305 def test_mpexec_failure(self): 

306 """Failure in one task should not stop other tasks""" 

307 

308 taskDef = TaskDefMock() 

309 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

310 qgraph = QuantumGraphMock([ 

311 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

312 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

313 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

314 ]) 

315 

316 qexec = QuantumExecutorMock(mp=True) 

317 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

318 with self.assertRaises(MPGraphExecutorError): 

319 mpexec.execute(qgraph, butler=None) 

320 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

321 

322 def test_mpexec_failure_dep(self): 

323 """Failure in one task should skip dependents""" 

324 

325 taskDef = TaskDefMock() 

326 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

327 qdata = [ 

328 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

329 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

330 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

331 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

332 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

333 ] 

334 qdata[2].dependencies.add(1) 

335 qdata[4].dependencies.add(3) 

336 qdata[4].dependencies.add(2) 

337 

338 qgraph = QuantumGraphMock(qdata) 

339 

340 qexec = QuantumExecutorMock(mp=True) 

341 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

342 with self.assertRaises(MPGraphExecutorError): 

343 mpexec.execute(qgraph, butler=None) 

344 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

345 

346 def test_mpexec_failure_failfast(self): 

347 """Fast fail stops quickly. 

348 

349 Timing delay of task #3 should be sufficient to process 

350 failure and raise exception. 

351 """ 

352 

353 taskDef = TaskDefMock() 

354 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

355 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

356 qdata = [ 

357 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

358 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

359 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

360 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3), 

361 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

362 ] 

363 qdata[1].dependencies.add(0) 

364 qdata[2].dependencies.add(1) 

365 qdata[4].dependencies.add(3) 

366 qdata[4].dependencies.add(2) 

367 

368 qgraph = QuantumGraphMock(qdata) 

369 

370 qexec = QuantumExecutorMock(mp=True) 

371 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

372 with self.assertRaises(MPGraphExecutorError): 

373 mpexec.execute(qgraph, butler=None) 

374 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

375 

376 def test_mpexec_num_fd(self): 

377 """Check that number of open files stays reasonable 

378 """ 

379 

380 taskDef = TaskDefMock() 

381 qgraph = QuantumGraphMock([ 

382 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20) 

383 ]) 

384 

385 this_proc = psutil.Process() 

386 num_fds_0 = this_proc.num_fds() 

387 

388 # run in multi-process mode, the order of results is not defined 

389 qexec = QuantumExecutorMock(mp=True) 

390 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

391 mpexec.execute(qgraph, butler=None) 

392 

393 num_fds_1 = this_proc.num_fds() 

394 # They should be the same but allow small growth just in case. 

395 # Without DM-26728 fix the difference would be equal to number of 

396 # quanta (20). 

397 self.assertLess(num_fds_1 - num_fds_0, 5) 

398 

399 

400if __name__ == "__main__": 400 ↛ 401line 400 didn't jump to line 401, because the condition on line 400 was never true

401 unittest.main()