Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import networkx as nx 

27from multiprocessing import Manager 

28import psutil 

29import sys 

30import time 

31from types import SimpleNamespace 

32import unittest 

33import warnings 

34 

35from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor 

36from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

37from lsst.pipe.base import NodeId 

38 

39 

40logging.basicConfig(level=logging.DEBUG) 

41 

42_LOG = logging.getLogger(__name__) 

43 

44 

45class QuantumExecutorMock(QuantumExecutor): 

46 """Mock class for QuantumExecutor 

47 """ 

48 def __init__(self, mp=False): 

49 self.quanta = [] 

50 if mp: 

51 # in multiprocess mode use shared list 

52 manager = Manager() 

53 self.quanta = manager.list() 

54 

55 def execute(self, taskDef, quantum, butler): 

56 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

57 if taskDef.taskClass: 

58 # only works for TaskMockMP class below 

59 taskDef.taskClass().runQuantum() 

60 self.quanta.append(quantum) 

61 

62 def getDataIds(self, field): 

63 """Returns values for dataId field for each visited quanta""" 

64 return [quantum.dataId[field] for quantum in self.quanta] 

65 

66 

67class QuantumIterDataMock: 

68 """Simple class to mock QuantumIterData. 

69 """ 

70 def __init__(self, index, taskDef, **dataId): 

71 self.index = index 

72 self.taskDef = taskDef 

73 self.quantum = SimpleNamespace(dataId=dataId) 

74 self.dependencies = set() 

75 self.nodeId = NodeId(index, "DummyBuildString") 

76 

77 

78class QuantumGraphMock: 

79 """Mock for quantum graph. 

80 """ 

81 def __init__(self, qdata): 

82 self._graph = nx.DiGraph() 

83 previous = qdata[0] 

84 for node in qdata[1:]: 

85 self._graph.add_edge(previous, node) 

86 previous = node 

87 

88 def __iter__(self): 

89 yield from nx.topological_sort(self._graph) 

90 

91 def __len__(self): 

92 return len(self._graph) 

93 

94 def findTaskDefByLabel(self, label): 

95 for q in self: 

96 if q.taskDef.label == label: 

97 return q.taskDef 

98 

99 def quantaForTask(self, taskDef): 

100 quanta = set() 

101 for q in self: 

102 if q.taskDef == taskDef: 

103 quanta.add(q) 

104 return quanta 

105 

106 @property 

107 def graph(self): 

108 return self._graph 

109 

110 def findCycle(self): 

111 return [] 

112 

113 def determineInputsToQuantumNode(self, node): 

114 result = set() 

115 for n in node.dependencies: 

116 for otherNode in self: 

117 if otherNode.index == n: 

118 result.add(otherNode) 

119 return result 

120 

121 

122class TaskMockMP: 

123 """Simple mock class for task supporting multiprocessing. 

124 """ 

125 canMultiprocess = True 

126 

127 def runQuantum(self): 

128 _LOG.debug("TaskMockMP.runQuantum") 

129 pass 

130 

131 

132class TaskMockFail: 

133 """Simple mock class for task which fails. 

134 """ 

135 canMultiprocess = True 

136 

137 def runQuantum(self): 

138 _LOG.debug("TaskMockFail.runQuantum") 

139 raise ValueError("expected failure") 

140 

141 

142class TaskMockSleep: 

143 """Simple mock class for task which "runs" for some time. 

144 """ 

145 canMultiprocess = True 

146 

147 def runQuantum(self): 

148 _LOG.debug("TaskMockSleep.runQuantum") 

149 time.sleep(5.) 

150 

151 

152class TaskMockLongSleep: 

153 """Simple mock class for task which "runs" for very long time. 

154 """ 

155 canMultiprocess = True 

156 

157 def runQuantum(self): 

158 _LOG.debug("TaskMockLongSleep.runQuantum") 

159 time.sleep(100.) 

160 

161 

162class TaskMockNoMP: 

163 """Simple mock class for task not supporting multiprocessing. 

164 """ 

165 canMultiprocess = False 

166 

167 

168class TaskDefMock: 

169 """Simple mock class for task definition in a pipeline. 

170 """ 

171 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

172 self.taskName = taskName 

173 self.config = config 

174 self.taskClass = taskClass 

175 self.label = label 

176 

177 def __str__(self): 

178 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

179 

180 

181class MPGraphExecutorTestCase(unittest.TestCase): 

182 """A test case for MPGraphExecutor class 

183 """ 

184 

185 def test_mpexec_nomp(self): 

186 """Make simple graph and execute""" 

187 

188 taskDef = TaskDefMock() 

189 qgraph = QuantumGraphMock([ 

190 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

191 ]) 

192 

193 # run in single-process mode 

194 qexec = QuantumExecutorMock() 

195 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

196 mpexec.execute(qgraph, butler=None) 

197 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

198 

199 def test_mpexec_mp(self): 

200 """Make simple graph and execute""" 

201 

202 taskDef = TaskDefMock() 

203 qgraph = QuantumGraphMock([ 

204 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

205 ]) 

206 

207 methods = ["spawn"] 

208 if sys.platform == "linux": 

209 methods.append("fork") 

210 methods.append("forkserver") 

211 

212 for method in methods: 

213 with self.subTest(startMethod=method): 

214 # run in multi-process mode, the order of results is not defined 

215 qexec = QuantumExecutorMock(mp=True) 

216 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method) 

217 mpexec.execute(qgraph, butler=None) 

218 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

219 

220 def test_mpexec_nompsupport(self): 

221 """Try to run MP for task that has no MP support which should fail 

222 """ 

223 

224 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

225 qgraph = QuantumGraphMock([ 

226 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

227 ]) 

228 

229 # run in multi-process mode 

230 qexec = QuantumExecutorMock() 

231 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

232 with self.assertRaises(MPGraphExecutorError): 

233 mpexec.execute(qgraph, butler=None) 

234 

235 def test_mpexec_fixup(self): 

236 """Make simple graph and execute, add dependencies by executing fixup code 

237 """ 

238 

239 taskDef = TaskDefMock() 

240 

241 for reverse in (False, True): 

242 qgraph = QuantumGraphMock([ 

243 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

244 ]) 

245 

246 qexec = QuantumExecutorMock() 

247 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

248 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, 

249 executionGraphFixup=fixup) 

250 mpexec.execute(qgraph, butler=None) 

251 

252 expected = [0, 1, 2] 

253 if reverse: 

254 expected = list(reversed(expected)) 

255 self.assertEqual(qexec.getDataIds("detector"), expected) 

256 

257 def test_mpexec_timeout(self): 

258 """Fail due to timeout""" 

259 

260 taskDef = TaskDefMock() 

261 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

262 qgraph = QuantumGraphMock([ 

263 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

264 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

265 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

266 ]) 

267 

268 # with failFast we'll get immediate MPTimeoutError 

269 qexec = QuantumExecutorMock(mp=True) 

270 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

271 with self.assertRaises(MPTimeoutError): 

272 mpexec.execute(qgraph, butler=None) 

273 

274 # with failFast=False exception happens after last task finishes 

275 qexec = QuantumExecutorMock(mp=True) 

276 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

277 with self.assertRaises(MPTimeoutError): 

278 mpexec.execute(qgraph, butler=None) 

279 # We expect two tasks (0 and 2) to finish successfully and one task to 

280 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

281 # finish on time, so expect more timeouts and issue a warning. 

282 detectorIds = set(qexec.getDataIds("detector")) 

283 self.assertLess(len(detectorIds), 3) 

284 if detectorIds != {0, 2}: 

285 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

286 

287 def test_mpexec_failure(self): 

288 """Failure in one task should not stop other tasks""" 

289 

290 taskDef = TaskDefMock() 

291 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

292 qgraph = QuantumGraphMock([ 

293 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

294 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

295 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

296 ]) 

297 

298 qexec = QuantumExecutorMock(mp=True) 

299 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

300 with self.assertRaises(MPGraphExecutorError): 

301 mpexec.execute(qgraph, butler=None) 

302 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

303 

304 def test_mpexec_failure_dep(self): 

305 """Failure in one task should skip dependents""" 

306 

307 taskDef = TaskDefMock() 

308 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

309 qdata = [ 

310 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

311 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

312 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

313 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

314 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

315 ] 

316 qdata[2].dependencies.add(1) 

317 qdata[4].dependencies.add(3) 

318 qdata[4].dependencies.add(2) 

319 

320 qgraph = QuantumGraphMock(qdata) 

321 

322 qexec = QuantumExecutorMock(mp=True) 

323 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

324 with self.assertRaises(MPGraphExecutorError): 

325 mpexec.execute(qgraph, butler=None) 

326 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

327 

328 def test_mpexec_failure_failfast(self): 

329 """Fast fail stops quickly. 

330 

331 Timing delay of task #3 should be sufficient to process 

332 failure and raise exception. 

333 """ 

334 

335 taskDef = TaskDefMock() 

336 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

337 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

338 qdata = [ 

339 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

340 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

341 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

342 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3), 

343 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

344 ] 

345 qdata[1].dependencies.add(0) 

346 qdata[2].dependencies.add(1) 

347 qdata[4].dependencies.add(3) 

348 qdata[4].dependencies.add(2) 

349 

350 qgraph = QuantumGraphMock(qdata) 

351 

352 qexec = QuantumExecutorMock(mp=True) 

353 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

354 with self.assertRaises(MPGraphExecutorError): 

355 mpexec.execute(qgraph, butler=None) 

356 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

357 

358 def test_mpexec_num_fd(self): 

359 """Check that number of open files stays reasonable 

360 """ 

361 

362 taskDef = TaskDefMock() 

363 qgraph = QuantumGraphMock([ 

364 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20) 

365 ]) 

366 

367 this_proc = psutil.Process() 

368 num_fds_0 = this_proc.num_fds() 

369 

370 # run in multi-process mode, the order of results is not defined 

371 qexec = QuantumExecutorMock(mp=True) 

372 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

373 mpexec.execute(qgraph, butler=None) 

374 

375 num_fds_1 = this_proc.num_fds() 

376 # They should be the same but allow small growth just in case. 

377 # Without DM-26728 fix the difference would be equal to number of 

378 # quanta (20). 

379 self.assertLess(num_fds_1 - num_fds_0, 5) 

380 

381 

382if __name__ == "__main__": 382 ↛ 383line 382 didn't jump to line 383, because the condition on line 382 was never true

383 unittest.main()