Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import networkx as nx 

27from multiprocessing import Manager 

28import psutil 

29import time 

30from types import SimpleNamespace 

31import unittest 

32import warnings 

33 

34from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor 

35from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

36from lsst.pipe.base import NodeId 

37 

38 

39logging.basicConfig(level=logging.DEBUG) 

40 

41_LOG = logging.getLogger(__name__) 

42 

43 

44class QuantumExecutorMock(QuantumExecutor): 

45 """Mock class for QuantumExecutor 

46 """ 

47 def __init__(self, mp=False): 

48 self.quanta = [] 

49 if mp: 

50 # in multiprocess mode use shared list 

51 manager = Manager() 

52 self.quanta = manager.list() 

53 

54 def execute(self, taskDef, quantum, butler): 

55 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

56 if taskDef.taskClass: 

57 # only works for TaskMockMP class below 

58 taskDef.taskClass().runQuantum() 

59 self.quanta.append(quantum) 

60 

61 def getDataIds(self, field): 

62 """Returns values for dataId field for each visited quanta""" 

63 return [quantum.dataId[field] for quantum in self.quanta] 

64 

65 

66class QuantumIterDataMock: 

67 """Simple class to mock QuantumIterData. 

68 """ 

69 def __init__(self, index, taskDef, **dataId): 

70 self.index = index 

71 self.taskDef = taskDef 

72 self.quantum = SimpleNamespace(dataId=dataId) 

73 self.dependencies = set() 

74 self.nodeId = NodeId(index, "DummyBuildString") 

75 

76 

77class QuantumGraphMock: 

78 """Mock for quantum graph. 

79 """ 

80 def __init__(self, qdata): 

81 self._graph = nx.DiGraph() 

82 previous = qdata[0] 

83 for node in qdata[1:]: 

84 self._graph.add_edge(previous, node) 

85 previous = node 

86 

87 def __iter__(self): 

88 yield from nx.topological_sort(self._graph) 

89 

90 def findTaskDefByLabel(self, label): 

91 for q in self: 

92 if q.taskDef.label == label: 

93 return q.taskDef 

94 

95 def quantaForTask(self, taskDef): 

96 quanta = set() 

97 for q in self: 

98 if q.taskDef == taskDef: 

99 quanta.add(q) 

100 return quanta 

101 

102 @property 

103 def graph(self): 

104 return self._graph 

105 

106 def findCycle(self): 

107 return [] 

108 

109 def determineInputsToQuantumNode(self, node): 

110 result = set() 

111 for n in node.dependencies: 

112 for otherNode in self: 

113 if otherNode.index == n: 

114 result.add(otherNode) 

115 return result 

116 

117 

118class TaskMockMP: 

119 """Simple mock class for task supporting multiprocessing. 

120 """ 

121 canMultiprocess = True 

122 

123 def runQuantum(self): 

124 _LOG.debug("TaskMockMP.runQuantum") 

125 pass 

126 

127 

128class TaskMockFail: 

129 """Simple mock class for task which fails. 

130 """ 

131 canMultiprocess = True 

132 

133 def runQuantum(self): 

134 _LOG.debug("TaskMockFail.runQuantum") 

135 raise ValueError("expected failure") 

136 

137 

138class TaskMockSleep: 

139 """Simple mock class for task which fails. 

140 """ 

141 canMultiprocess = True 

142 

143 def runQuantum(self): 

144 _LOG.debug("TaskMockSleep.runQuantum") 

145 time.sleep(5.) 

146 

147 

148class TaskMockNoMP: 

149 """Simple mock class for task not supporting multiprocessing. 

150 """ 

151 canMultiprocess = False 

152 

153 

154class TaskDefMock: 

155 """Simple mock class for task definition in a pipeline. 

156 """ 

157 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

158 self.taskName = taskName 

159 self.config = config 

160 self.taskClass = taskClass 

161 self.label = label 

162 

163 def __str__(self): 

164 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

165 

166 

167class MPGraphExecutorTestCase(unittest.TestCase): 

168 """A test case for MPGraphExecutor class 

169 """ 

170 

171 def test_mpexec_nomp(self): 

172 """Make simple graph and execute""" 

173 

174 taskDef = TaskDefMock() 

175 qgraph = QuantumGraphMock([ 

176 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

177 ]) 

178 

179 # run in single-process mode 

180 qexec = QuantumExecutorMock() 

181 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

182 mpexec.execute(qgraph, butler=None) 

183 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

184 

185 def test_mpexec_mp(self): 

186 """Make simple graph and execute""" 

187 

188 taskDef = TaskDefMock() 

189 qgraph = QuantumGraphMock([ 

190 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

191 ]) 

192 

193 # run in multi-process mode, the order of results is not defined 

194 qexec = QuantumExecutorMock(mp=True) 

195 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

196 mpexec.execute(qgraph, butler=None) 

197 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

198 

199 def test_mpexec_nompsupport(self): 

200 """Try to run MP for task that has no MP support which should fail 

201 """ 

202 

203 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

204 qgraph = QuantumGraphMock([ 

205 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

206 ]) 

207 

208 # run in multi-process mode 

209 qexec = QuantumExecutorMock() 

210 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

211 with self.assertRaises(MPGraphExecutorError): 

212 mpexec.execute(qgraph, butler=None) 

213 

214 def test_mpexec_fixup(self): 

215 """Make simple graph and execute, add dependencies by executing fixup code 

216 """ 

217 

218 taskDef = TaskDefMock() 

219 

220 for reverse in (False, True): 

221 qgraph = QuantumGraphMock([ 

222 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

223 ]) 

224 

225 qexec = QuantumExecutorMock() 

226 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

227 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, 

228 executionGraphFixup=fixup) 

229 mpexec.execute(qgraph, butler=None) 

230 

231 expected = [0, 1, 2] 

232 if reverse: 

233 expected = list(reversed(expected)) 

234 self.assertEqual(qexec.getDataIds("detector"), expected) 

235 

236 def test_mpexec_timeout(self): 

237 """Fail due to timeout""" 

238 

239 taskDef = TaskDefMock() 

240 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

241 qgraph = QuantumGraphMock([ 

242 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

243 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

244 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

245 ]) 

246 

247 # with failFast we'll get immediate MPTimeoutError 

248 qexec = QuantumExecutorMock(mp=True) 

249 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

250 with self.assertRaises(MPTimeoutError): 

251 mpexec.execute(qgraph, butler=None) 

252 

253 # with failFast=False exception happens after last task finishes 

254 qexec = QuantumExecutorMock(mp=True) 

255 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

256 with self.assertRaises(MPTimeoutError): 

257 mpexec.execute(qgraph, butler=None) 

258 # We expect two tasks (0 and 2) to finish successfully and one task to 

259 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

260 # finish on time, so expect more timeouts and issue a warning. 

261 detectorIds = set(qexec.getDataIds("detector")) 

262 self.assertLess(len(detectorIds), 3) 

263 if detectorIds != {0, 2}: 

264 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

265 

266 def test_mpexec_failure(self): 

267 """Failure in one task should not stop other tasks""" 

268 

269 taskDef = TaskDefMock() 

270 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

271 qgraph = QuantumGraphMock([ 

272 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

273 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

274 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

275 ]) 

276 

277 qexec = QuantumExecutorMock(mp=True) 

278 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

279 with self.assertRaises(MPGraphExecutorError): 

280 mpexec.execute(qgraph, butler=None) 

281 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

282 

283 def test_mpexec_failure_dep(self): 

284 """Failure in one task should skip dependents""" 

285 

286 taskDef = TaskDefMock() 

287 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

288 qdata = [ 

289 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

290 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

291 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

292 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

293 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

294 ] 

295 qdata[2].dependencies.add(1) 

296 qdata[4].dependencies.add(3) 

297 qdata[4].dependencies.add(2) 

298 

299 qgraph = QuantumGraphMock(qdata) 

300 

301 qexec = QuantumExecutorMock(mp=True) 

302 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

303 with self.assertRaises(MPGraphExecutorError): 

304 mpexec.execute(qgraph, butler=None) 

305 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

306 

307 def test_mpexec_failure_failfast(self): 

308 """Fast fail stops quickly. 

309 

310 Timing delay of task #3 should be sufficient to process 

311 failure and raise exception. 

312 """ 

313 

314 taskDef = TaskDefMock() 

315 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

316 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

317 qdata = [ 

318 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

319 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

320 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

321 QuantumIterDataMock(index=3, taskDef=taskDefSleep, detector=3), 

322 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

323 ] 

324 qdata[1].dependencies.add(0) 

325 qdata[2].dependencies.add(1) 

326 qdata[4].dependencies.add(3) 

327 qdata[4].dependencies.add(2) 

328 

329 qgraph = QuantumGraphMock(qdata) 

330 

331 qexec = QuantumExecutorMock(mp=True) 

332 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

333 with self.assertRaises(MPGraphExecutorError): 

334 mpexec.execute(qgraph, butler=None) 

335 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

336 

337 def test_mpexec_num_fd(self): 

338 """Check that number of open files stays reasonable 

339 """ 

340 

341 taskDef = TaskDefMock() 

342 qgraph = QuantumGraphMock([ 

343 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20) 

344 ]) 

345 

346 this_proc = psutil.Process() 

347 num_fds_0 = this_proc.num_fds() 

348 

349 # run in multi-process mode, the order of results is not defined 

350 qexec = QuantumExecutorMock(mp=True) 

351 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

352 mpexec.execute(qgraph, butler=None) 

353 

354 num_fds_1 = this_proc.num_fds() 

355 # They should be the same but allow small growth just in case. 

356 # Without DM-26728 fix the difference would be equal to number of 

357 # quanta (20). 

358 self.assertLess(num_fds_1 - num_fds_0, 5) 

359 

360 

361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true

362 unittest.main()