Coverage for tests/test_executors.py: 25%

212 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:04 -0800

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import networkx as nx 

27from multiprocessing import Manager 

28import psutil 

29import sys 

30import time 

31import unittest 

32import warnings 

33 

34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor 

35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

36from lsst.pipe.base import NodeId 

37 

38 

39logging.basicConfig(level=logging.DEBUG) 

40 

41_LOG = logging.getLogger(__name__) 

42 

43 

44class QuantumExecutorMock(QuantumExecutor): 

45 """Mock class for QuantumExecutor 

46 """ 

47 def __init__(self, mp=False): 

48 self.quanta = [] 

49 if mp: 

50 # in multiprocess mode use shared list 

51 manager = Manager() 

52 self.quanta = manager.list() 

53 

54 def execute(self, taskDef, quantum, butler): 

55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

56 if taskDef.taskClass: 

57 # only works for TaskMockMP class below 

58 taskDef.taskClass().runQuantum() 

59 self.quanta.append(quantum) 

60 

61 def getDataIds(self, field): 

62 """Returns values for dataId field for each visited quanta""" 

63 return [quantum.dataId[field] for quantum in self.quanta] 

64 

65 

66class QuantumMock: 

67 def __init__(self, dataId): 

68 self.dataId = dataId 

69 

70 def __eq__(self, other): 

71 return self.dataId == other.dataId 

72 

73 def __hash__(self): 

74 # dict.__eq__ is order-insensitive 

75 return hash(tuple(sorted(kv for kv in self.dataId.items()))) 

76 

77 

78class QuantumIterDataMock: 

79 """Simple class to mock QuantumIterData. 

80 """ 

81 def __init__(self, index, taskDef, **dataId): 

82 self.index = index 

83 self.taskDef = taskDef 

84 self.quantum = QuantumMock(dataId) 

85 self.dependencies = set() 

86 self.nodeId = NodeId(index, "DummyBuildString") 

87 

88 

89class QuantumGraphMock: 

90 """Mock for quantum graph. 

91 """ 

92 def __init__(self, qdata): 

93 self._graph = nx.DiGraph() 

94 previous = qdata[0] 

95 for node in qdata[1:]: 

96 self._graph.add_edge(previous, node) 

97 previous = node 

98 

99 def __iter__(self): 

100 yield from nx.topological_sort(self._graph) 

101 

102 def __len__(self): 

103 return len(self._graph) 

104 

105 def findTaskDefByLabel(self, label): 

106 for q in self: 

107 if q.taskDef.label == label: 

108 return q.taskDef 

109 

110 def getQuantaForTask(self, taskDef): 

111 nodes = self.getNodesForTask(taskDef) 

112 return {q.quantum for q in nodes} 

113 

114 def getNodesForTask(self, taskDef): 

115 quanta = set() 

116 for q in self: 

117 if q.taskDef == taskDef: 

118 quanta.add(q) 

119 return quanta 

120 

121 @property 

122 def graph(self): 

123 return self._graph 

124 

125 def findCycle(self): 

126 return [] 

127 

128 def determineInputsToQuantumNode(self, node): 

129 result = set() 

130 for n in node.dependencies: 

131 for otherNode in self: 

132 if otherNode.index == n: 

133 result.add(otherNode) 

134 return result 

135 

136 

137class TaskMockMP: 

138 """Simple mock class for task supporting multiprocessing. 

139 """ 

140 canMultiprocess = True 

141 

142 def runQuantum(self): 

143 _LOG.debug("TaskMockMP.runQuantum") 

144 pass 

145 

146 

147class TaskMockFail: 

148 """Simple mock class for task which fails. 

149 """ 

150 canMultiprocess = True 

151 

152 def runQuantum(self): 

153 _LOG.debug("TaskMockFail.runQuantum") 

154 raise ValueError("expected failure") 

155 

156 

157class TaskMockSleep: 

158 """Simple mock class for task which "runs" for some time. 

159 """ 

160 canMultiprocess = True 

161 

162 def runQuantum(self): 

163 _LOG.debug("TaskMockSleep.runQuantum") 

164 time.sleep(5.) 

165 

166 

167class TaskMockLongSleep: 

168 """Simple mock class for task which "runs" for very long time. 

169 """ 

170 canMultiprocess = True 

171 

172 def runQuantum(self): 

173 _LOG.debug("TaskMockLongSleep.runQuantum") 

174 time.sleep(100.) 

175 

176 

177class TaskMockNoMP: 

178 """Simple mock class for task not supporting multiprocessing. 

179 """ 

180 canMultiprocess = False 

181 

182 

183class TaskDefMock: 

184 """Simple mock class for task definition in a pipeline. 

185 """ 

186 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

187 self.taskName = taskName 

188 self.config = config 

189 self.taskClass = taskClass 

190 self.label = label 

191 

192 def __str__(self): 

193 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

194 

195 

196class MPGraphExecutorTestCase(unittest.TestCase): 

197 """A test case for MPGraphExecutor class 

198 """ 

199 

200 def test_mpexec_nomp(self): 

201 """Make simple graph and execute""" 

202 

203 taskDef = TaskDefMock() 

204 qgraph = QuantumGraphMock([ 

205 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

206 ]) 

207 

208 # run in single-process mode 

209 qexec = QuantumExecutorMock() 

210 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

211 mpexec.execute(qgraph, butler=None) 

212 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

213 

214 def test_mpexec_mp(self): 

215 """Make simple graph and execute""" 

216 

217 taskDef = TaskDefMock() 

218 qgraph = QuantumGraphMock([ 

219 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

220 ]) 

221 

222 methods = ["spawn"] 

223 if sys.platform == "linux": 

224 methods.append("fork") 

225 methods.append("forkserver") 

226 

227 for method in methods: 

228 with self.subTest(startMethod=method): 

229 # Run in multi-process mode, the order of results is not 

230 # defined. 

231 qexec = QuantumExecutorMock(mp=True) 

232 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method) 

233 mpexec.execute(qgraph, butler=None) 

234 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

235 

236 def test_mpexec_nompsupport(self): 

237 """Try to run MP for task that has no MP support which should fail 

238 """ 

239 

240 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

241 qgraph = QuantumGraphMock([ 

242 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

243 ]) 

244 

245 # run in multi-process mode 

246 qexec = QuantumExecutorMock() 

247 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

248 with self.assertRaises(MPGraphExecutorError): 

249 mpexec.execute(qgraph, butler=None) 

250 

251 def test_mpexec_fixup(self): 

252 """Make simple graph and execute, add dependencies by executing fixup 

253 code. 

254 """ 

255 

256 taskDef = TaskDefMock() 

257 

258 for reverse in (False, True): 

259 qgraph = QuantumGraphMock([ 

260 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

261 ]) 

262 

263 qexec = QuantumExecutorMock() 

264 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

265 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, 

266 executionGraphFixup=fixup) 

267 mpexec.execute(qgraph, butler=None) 

268 

269 expected = [0, 1, 2] 

270 if reverse: 

271 expected = list(reversed(expected)) 

272 self.assertEqual(qexec.getDataIds("detector"), expected) 

273 

274 def test_mpexec_timeout(self): 

275 """Fail due to timeout""" 

276 

277 taskDef = TaskDefMock() 

278 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

279 qgraph = QuantumGraphMock([ 

280 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

281 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

282 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

283 ]) 

284 

285 # with failFast we'll get immediate MPTimeoutError 

286 qexec = QuantumExecutorMock(mp=True) 

287 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

288 with self.assertRaises(MPTimeoutError): 

289 mpexec.execute(qgraph, butler=None) 

290 

291 # with failFast=False exception happens after last task finishes 

292 qexec = QuantumExecutorMock(mp=True) 

293 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

294 with self.assertRaises(MPTimeoutError): 

295 mpexec.execute(qgraph, butler=None) 

296 # We expect two tasks (0 and 2) to finish successfully and one task to 

297 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

298 # finish on time, so expect more timeouts and issue a warning. 

299 detectorIds = set(qexec.getDataIds("detector")) 

300 self.assertLess(len(detectorIds), 3) 

301 if detectorIds != {0, 2}: 

302 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

303 

304 def test_mpexec_failure(self): 

305 """Failure in one task should not stop other tasks""" 

306 

307 taskDef = TaskDefMock() 

308 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

309 qgraph = QuantumGraphMock([ 

310 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

311 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

312 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

313 ]) 

314 

315 qexec = QuantumExecutorMock(mp=True) 

316 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

317 with self.assertRaises(MPGraphExecutorError): 

318 mpexec.execute(qgraph, butler=None) 

319 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

320 

321 def test_mpexec_failure_dep(self): 

322 """Failure in one task should skip dependents""" 

323 

324 taskDef = TaskDefMock() 

325 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

326 qdata = [ 

327 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

328 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

329 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

330 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

331 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

332 ] 

333 qdata[2].dependencies.add(1) 

334 qdata[4].dependencies.add(3) 

335 qdata[4].dependencies.add(2) 

336 

337 qgraph = QuantumGraphMock(qdata) 

338 

339 qexec = QuantumExecutorMock(mp=True) 

340 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

341 with self.assertRaises(MPGraphExecutorError): 

342 mpexec.execute(qgraph, butler=None) 

343 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

344 

345 def test_mpexec_failure_failfast(self): 

346 """Fast fail stops quickly. 

347 

348 Timing delay of task #3 should be sufficient to process 

349 failure and raise exception. 

350 """ 

351 

352 taskDef = TaskDefMock() 

353 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

354 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

355 qdata = [ 

356 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

357 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

358 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

359 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3), 

360 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

361 ] 

362 qdata[1].dependencies.add(0) 

363 qdata[2].dependencies.add(1) 

364 qdata[4].dependencies.add(3) 

365 qdata[4].dependencies.add(2) 

366 

367 qgraph = QuantumGraphMock(qdata) 

368 

369 qexec = QuantumExecutorMock(mp=True) 

370 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

371 with self.assertRaises(MPGraphExecutorError): 

372 mpexec.execute(qgraph, butler=None) 

373 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

374 

375 def test_mpexec_num_fd(self): 

376 """Check that number of open files stays reasonable 

377 """ 

378 

379 taskDef = TaskDefMock() 

380 qgraph = QuantumGraphMock([ 

381 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20) 

382 ]) 

383 

384 this_proc = psutil.Process() 

385 num_fds_0 = this_proc.num_fds() 

386 

387 # run in multi-process mode, the order of results is not defined 

388 qexec = QuantumExecutorMock(mp=True) 

389 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

390 mpexec.execute(qgraph, butler=None) 

391 

392 num_fds_1 = this_proc.num_fds() 

393 # They should be the same but allow small growth just in case. 

394 # Without DM-26728 fix the difference would be equal to number of 

395 # quanta (20). 

396 self.assertLess(num_fds_1 - num_fds_0, 5) 

397 

398 

399if __name__ == "__main__": 399 ↛ 400line 399 didn't jump to line 400, because the condition on line 399 was never true

400 unittest.main()