Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26from multiprocessing import Manager 

27import psutil 

28import time 

29from types import SimpleNamespace 

30import unittest 

31 

32from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor 

33from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

34 

35 

36logging.basicConfig(level=logging.DEBUG) 

37 

38_LOG = logging.getLogger(__name__) 

39 

40 

41class QuantumExecutorMock(QuantumExecutor): 

42 """Mock class for QuantumExecutor 

43 """ 

44 def __init__(self, mp=False): 

45 self.quanta = [] 

46 if mp: 

47 # in multiprocess mode use shared list 

48 manager = Manager() 

49 self.quanta = manager.list() 

50 

51 def execute(self, taskDef, quantum, butler): 

52 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

53 if taskDef.taskClass: 

54 # only works for TaskMockMP class below 

55 taskDef.taskClass().runQuantum() 

56 self.quanta.append(quantum) 

57 

58 def getDataIds(self, field): 

59 """Returns values for dataId field for each visited quanta""" 

60 return [quantum.dataId[field] for quantum in self.quanta] 

61 

62 

63class QuantumIterDataMock: 

64 """Simple class to mock QuantumIterData. 

65 """ 

66 def __init__(self, index, taskDef, **dataId): 

67 self.index = index 

68 self.taskDef = taskDef 

69 self.quantum = SimpleNamespace(dataId=dataId) 

70 self.dependencies = set() 

71 

72 

73class QuantumGraphMock: 

74 """Mock for quantum graph. 

75 """ 

76 def __init__(self, qdata): 

77 self.qdata = qdata 

78 

79 def traverse(self): 

80 return self.qdata 

81 

82 

83class TaskMockMP: 

84 """Simple mock class for task supporting multiprocessing. 

85 """ 

86 canMultiprocess = True 

87 

88 def runQuantum(self): 

89 _LOG.debug("TaskMockMP.runQuantum") 

90 pass 

91 

92 

93class TaskMockFail: 

94 """Simple mock class for task which fails. 

95 """ 

96 canMultiprocess = True 

97 

98 def runQuantum(self): 

99 _LOG.debug("TaskMockFail.runQuantum") 

100 raise ValueError("expected failure") 

101 

102 

103class TaskMockSleep: 

104 """Simple mock class for task which fails. 

105 """ 

106 canMultiprocess = True 

107 

108 def runQuantum(self): 

109 _LOG.debug("TaskMockSleep.runQuantum") 

110 time.sleep(3.) 

111 

112 

113class TaskMockNoMP: 

114 """Simple mock class for task not supporting multiprocessing. 

115 """ 

116 canMultiprocess = False 

117 

118 

119class TaskDefMock: 

120 """Simple mock class for task definition in a pipeline. 

121 """ 

122 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

123 self.taskName = taskName 

124 self.config = config 

125 self.taskClass = taskClass 

126 self.label = label 

127 

128 def __str__(self): 

129 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

130 

131 

132class MPGraphExecutorTestCase(unittest.TestCase): 

133 """A test case for MPGraphExecutor class 

134 """ 

135 

136 def test_mpexec_nomp(self): 

137 """Make simple graph and execute""" 

138 

139 taskDef = TaskDefMock() 

140 qgraph = QuantumGraphMock([ 

141 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

142 ]) 

143 

144 # run in single-process mode 

145 qexec = QuantumExecutorMock() 

146 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

147 mpexec.execute(qgraph, butler=None) 

148 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

149 

150 def test_mpexec_mp(self): 

151 """Make simple graph and execute""" 

152 

153 taskDef = TaskDefMock() 

154 qgraph = QuantumGraphMock([ 

155 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

156 ]) 

157 

158 # run in multi-process mode, the order of results is not defined 

159 qexec = QuantumExecutorMock(mp=True) 

160 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

161 mpexec.execute(qgraph, butler=None) 

162 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

163 

164 def test_mpexec_nompsupport(self): 

165 """Try to run MP for task that has no MP support which should fail 

166 """ 

167 

168 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

169 qgraph = QuantumGraphMock([ 

170 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

171 ]) 

172 

173 # run in multi-process mode 

174 qexec = QuantumExecutorMock() 

175 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

176 with self.assertRaises(MPGraphExecutorError): 

177 mpexec.execute(qgraph, butler=None) 

178 

179 def test_mpexec_fixup(self): 

180 """Make simple graph and execute, add dependencies by executing fixup code 

181 """ 

182 

183 taskDef = TaskDefMock() 

184 

185 for reverse in (False, True): 

186 

187 qgraph = QuantumGraphMock([ 

188 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3) 

189 ]) 

190 

191 qexec = QuantumExecutorMock() 

192 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

193 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, 

194 executionGraphFixup=fixup) 

195 mpexec.execute(qgraph, butler=None) 

196 

197 expected = [0, 1, 2] 

198 if reverse: 

199 expected = list(reversed(expected)) 

200 self.assertEqual(qexec.getDataIds("detector"), expected) 

201 

202 def test_mpexec_timeout(self): 

203 """Fail due to timeout""" 

204 

205 taskDef = TaskDefMock() 

206 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

207 qgraph = QuantumGraphMock([ 

208 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

209 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

210 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

211 ]) 

212 

213 # with failFast we'll get immediate MPTimeoutError 

214 qexec = QuantumExecutorMock(mp=True) 

215 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

216 with self.assertRaises(MPTimeoutError): 

217 mpexec.execute(qgraph, butler=None) 

218 

219 # with failFast=False exception happens after last task finishes 

220 qexec = QuantumExecutorMock(mp=True) 

221 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=False) 

222 with self.assertRaises(MPTimeoutError): 

223 mpexec.execute(qgraph, butler=None) 

224 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

225 

226 def test_mpexec_failure(self): 

227 """Failure in one task should not stop other tasks""" 

228 

229 taskDef = TaskDefMock() 

230 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

231 qgraph = QuantumGraphMock([ 

232 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

233 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

234 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

235 ]) 

236 

237 qexec = QuantumExecutorMock(mp=True) 

238 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

239 with self.assertRaises(MPGraphExecutorError): 

240 mpexec.execute(qgraph, butler=None) 

241 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

242 

243 def test_mpexec_failure_dep(self): 

244 """Failure in one task should skip dependents""" 

245 

246 taskDef = TaskDefMock() 

247 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

248 qdata = [ 

249 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

250 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

251 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

252 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

253 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

254 ] 

255 qdata[2].dependencies.add(1) 

256 qdata[4].dependencies.add(3) 

257 qdata[4].dependencies.add(2) 

258 

259 qgraph = QuantumGraphMock(qdata) 

260 

261 qexec = QuantumExecutorMock(mp=True) 

262 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

263 with self.assertRaises(MPGraphExecutorError): 

264 mpexec.execute(qgraph, butler=None) 

265 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

266 

267 def test_mpexec_failure_failfast(self): 

268 """Fast fail stops quickly. 

269 

270 Timing delay of task #3 should be sufficient to process 

271 failure and raise exception. 

272 """ 

273 

274 taskDef = TaskDefMock() 

275 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

276 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

277 qdata = [ 

278 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

279 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

280 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

281 QuantumIterDataMock(index=3, taskDef=taskDefSleep, detector=3), 

282 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

283 ] 

284 qdata[1].dependencies.add(0) 

285 qdata[2].dependencies.add(1) 

286 qdata[4].dependencies.add(3) 

287 qdata[4].dependencies.add(2) 

288 

289 qgraph = QuantumGraphMock(qdata) 

290 

291 qexec = QuantumExecutorMock(mp=True) 

292 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

293 with self.assertRaises(MPGraphExecutorError): 

294 mpexec.execute(qgraph, butler=None) 

295 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

296 

297 def test_mpexec_num_fd(self): 

298 """Check that number of open files stays reasonable 

299 """ 

300 

301 taskDef = TaskDefMock() 

302 qgraph = QuantumGraphMock([ 

303 QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20) 

304 ]) 

305 

306 this_proc = psutil.Process() 

307 num_fds_0 = this_proc.num_fds() 

308 

309 # run in multi-process mode, the order of results is not defined 

310 qexec = QuantumExecutorMock(mp=True) 

311 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

312 mpexec.execute(qgraph, butler=None) 

313 

314 num_fds_1 = this_proc.num_fds() 

315 # They should be the same but allow small growth just in case. 

316 # Without DM-26728 fix the difference would be equal to number of 

317 # quanta (20). 

318 self.assertLess(num_fds_1 - num_fds_0, 5) 

319 

320 

321if __name__ == "__main__": 321 ↛ 322line 321 didn't jump to line 322, because the condition on line 321 was never true

322 unittest.main()