Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import networkx as nx 

27from multiprocessing import Manager 

28import psutil 

29import time 

30from types import SimpleNamespace 

31import unittest 

32import warnings 

33 

34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor 

35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

36from lsst.pipe.base import NodeId 

37 

38 

39logging.basicConfig(level=logging.DEBUG) 

40 

41_LOG = logging.getLogger(__name__) 

42 

43 

44class QuantumExecutorMock(QuantumExecutor): 

45 """Mock class for QuantumExecutor 

46 """ 

47 def __init__(self, mp=False): 

48 self.quanta = [] 

49 if mp: 

50 # in multiprocess mode use shared list 

51 manager = Manager() 

52 self.quanta = manager.list() 

53 

54 def execute(self, taskDef, quantum, butler): 

55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

56 if taskDef.taskClass: 

57 # only works for TaskMockMP class below 

58 taskDef.taskClass().runQuantum() 

59 self.quanta.append(quantum) 

60 

61 def getDataIds(self, field): 

62 """Returns values for dataId field for each visited quanta""" 

63 return [quantum.dataId[field] for quantum in self.quanta] 

64 

65 

66class QuantumIterDataMock: 

67 """Simple class to mock QuantumIterData. 

68 """ 

69 def __init__(self, index, taskDef, **dataId): 

70 self.index = index 

71 self.taskDef = taskDef 

72 self.quantum = SimpleNamespace(dataId=dataId) 

73 self.dependencies = set() 

74 self.nodeId = NodeId(index, "DummyBuildString") 

75 

76 

77class QuantumGraphMock: 

78 """Mock for quantum graph. 

79 """ 

80 def __init__(self, qdata): 

81 self._graph = nx.DiGraph() 

82 previous = qdata[0] 

83 for node in qdata[1:]: 

84 self._graph.add_edge(previous, node) 

85 previous = node 

86 

87 def __iter__(self): 

88 yield from nx.topological_sort(self._graph) 

89 

90 def __len__(self): 

91 return len(self._graph) 

92 

93 def findTaskDefByLabel(self, label): 

94 for q in self: 

95 if q.taskDef.label == label: 

96 return q.taskDef 

97 

98 def quantaForTask(self, taskDef): 

99 quanta = set() 

100 for q in self: 

101 if q.taskDef == taskDef: 

102 quanta.add(q) 

103 return quanta 

104 

105 @property 

106 def graph(self): 

107 return self._graph 

108 

109 def findCycle(self): 

110 return [] 

111 

112 def determineInputsToQuantumNode(self, node): 

113 result = set() 

114 for n in node.dependencies: 

115 for otherNode in self: 

116 if otherNode.index == n: 

117 result.add(otherNode) 

118 return result 

119 

120 

121class TaskMockMP: 

122 """Simple mock class for task supporting multiprocessing. 

123 """ 

124 canMultiprocess = True 

125 

126 def runQuantum(self): 

127 _LOG.debug("TaskMockMP.runQuantum") 

128 pass 

129 

130 

131class TaskMockFail: 

132 """Simple mock class for task which fails. 

133 """ 

134 canMultiprocess = True 

135 

136 def runQuantum(self): 

137 _LOG.debug("TaskMockFail.runQuantum") 

138 raise ValueError("expected failure") 

139 

140 

141class TaskMockSleep: 

142 """Simple mock class for task which "runs" for some time. 

143 """ 

144 canMultiprocess = True 

145 

146 def runQuantum(self): 

147 _LOG.debug("TaskMockSleep.runQuantum") 

148 time.sleep(5.) 

149 

150 

151class TaskMockLongSleep: 

152 """Simple mock class for task which "runs" for very long time. 

153 """ 

154 canMultiprocess = True 

155 

156 def runQuantum(self): 

157 _LOG.debug("TaskMockLongSleep.runQuantum") 

158 time.sleep(100.) 

159 

160 

161class TaskMockNoMP: 

162 """Simple mock class for task not supporting multiprocessing. 

163 """ 

164 canMultiprocess = False 

165 

166 

167class TaskDefMock: 

168 """Simple mock class for task definition in a pipeline. 

169 """ 

170 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

171 self.taskName = taskName 

172 self.config = config 

173 self.taskClass = taskClass 

174 self.label = label 

175 

176 def __str__(self): 

177 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

178 

179 

180class MPGraphExecutorTestCase(unittest.TestCase): 

181 """A test case for MPGraphExecutor class 

182 """ 

183 

184 def test_mpexec_nomp(self): 

185 """Make simple graph and execute""" 

186 

187 taskDef = TaskDefMock() 

188 qgraph = QuantumGraphMock([ 

189 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

190 ]) 

191 

192 # run in single-process mode 

193 qexec = QuantumExecutorMock() 

194 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

195 mpexec.execute(qgraph, butler=None) 

196 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

197 

198 def test_mpexec_mp(self): 

199 """Make simple graph and execute""" 

200 

201 taskDef = TaskDefMock() 

202 qgraph = QuantumGraphMock([ 

203 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

204 ]) 

205 

206 # run in multi-process mode, the order of results is not defined 

207 qexec = QuantumExecutorMock(mp=True) 

208 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

209 mpexec.execute(qgraph, butler=None) 

210 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

211 

212 def test_mpexec_nompsupport(self): 

213 """Try to run MP for task that has no MP support which should fail 

214 """ 

215 

216 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

217 qgraph = QuantumGraphMock([ 

218 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

219 ]) 

220 

221 # run in multi-process mode 

222 qexec = QuantumExecutorMock() 

223 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

224 with self.assertRaises(MPGraphExecutorError): 

225 mpexec.execute(qgraph, butler=None) 

226 

227 def test_mpexec_fixup(self): 

228 """Make simple graph and execute, add dependencies by executing fixup code 

229 """ 

230 

231 taskDef = TaskDefMock() 

232 

233 for reverse in (False, True): 

234 qgraph = QuantumGraphMock([ 

235 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

236 ]) 

237 

238 qexec = QuantumExecutorMock() 

239 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

240 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, 

241 executionGraphFixup=fixup) 

242 mpexec.execute(qgraph, butler=None) 

243 

244 expected = [0, 1, 2] 

245 if reverse: 

246 expected = list(reversed(expected)) 

247 self.assertEqual(qexec.getDataIds("detector"), expected) 

248 

249 def test_mpexec_timeout(self): 

250 """Fail due to timeout""" 

251 

252 taskDef = TaskDefMock() 

253 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

254 qgraph = QuantumGraphMock([ 

255 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

256 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

257 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

258 ]) 

259 

260 # with failFast we'll get immediate MPTimeoutError 

261 qexec = QuantumExecutorMock(mp=True) 

262 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

263 with self.assertRaises(MPTimeoutError): 

264 mpexec.execute(qgraph, butler=None) 

265 

266 # with failFast=False exception happens after last task finishes 

267 qexec = QuantumExecutorMock(mp=True) 

268 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

269 with self.assertRaises(MPTimeoutError): 

270 mpexec.execute(qgraph, butler=None) 

271 # We expect two tasks (0 and 2) to finish successfully and one task to 

272 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

273 # finish on time, so expect more timeouts and issue a warning. 

274 detectorIds = set(qexec.getDataIds("detector")) 

275 self.assertLess(len(detectorIds), 3) 

276 if detectorIds != {0, 2}: 

277 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

278 

279 def test_mpexec_failure(self): 

280 """Failure in one task should not stop other tasks""" 

281 

282 taskDef = TaskDefMock() 

283 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

284 qgraph = QuantumGraphMock([ 

285 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

286 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

287 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

288 ]) 

289 

290 qexec = QuantumExecutorMock(mp=True) 

291 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

292 with self.assertRaises(MPGraphExecutorError): 

293 mpexec.execute(qgraph, butler=None) 

294 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

295 

296 def test_mpexec_failure_dep(self): 

297 """Failure in one task should skip dependents""" 

298 

299 taskDef = TaskDefMock() 

300 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

301 qdata = [ 

302 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

303 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

304 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

305 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

306 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

307 ] 

308 qdata[2].dependencies.add(1) 

309 qdata[4].dependencies.add(3) 

310 qdata[4].dependencies.add(2) 

311 

312 qgraph = QuantumGraphMock(qdata) 

313 

314 qexec = QuantumExecutorMock(mp=True) 

315 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

316 with self.assertRaises(MPGraphExecutorError): 

317 mpexec.execute(qgraph, butler=None) 

318 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

319 

320 def test_mpexec_failure_failfast(self): 

321 """Fast fail stops quickly. 

322 

323 Timing delay of task #3 should be sufficient to process 

324 failure and raise exception. 

325 """ 

326 

327 taskDef = TaskDefMock() 

328 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

329 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

330 qdata = [ 

331 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

332 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

333 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

334 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3), 

335 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

336 ] 

337 qdata[1].dependencies.add(0) 

338 qdata[2].dependencies.add(1) 

339 qdata[4].dependencies.add(3) 

340 qdata[4].dependencies.add(2) 

341 

342 qgraph = QuantumGraphMock(qdata) 

343 

344 qexec = QuantumExecutorMock(mp=True) 

345 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

346 with self.assertRaises(MPGraphExecutorError): 

347 mpexec.execute(qgraph, butler=None) 

348 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

349 

350 def test_mpexec_num_fd(self): 

351 """Check that number of open files stays reasonable 

352 """ 

353 

354 taskDef = TaskDefMock() 

355 qgraph = QuantumGraphMock([ 

356 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20) 

357 ]) 

358 

359 this_proc = psutil.Process() 

360 num_fds_0 = this_proc.num_fds() 

361 

362 # run in multi-process mode, the order of results is not defined 

363 qexec = QuantumExecutorMock(mp=True) 

364 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

365 mpexec.execute(qgraph, butler=None) 

366 

367 num_fds_1 = this_proc.num_fds() 

368 # They should be the same but allow small growth just in case. 

369 # Without DM-26728 fix the difference would be equal to number of 

370 # quanta (20). 

371 self.assertLess(num_fds_1 - num_fds_0, 5) 

372 

373 

374if __name__ == "__main__": 374 ↛ 375line 374 didn't jump to line 375, because the condition on line 374 was never true

375 unittest.main()