Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['MPGraphExecutor'] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28import multiprocessing 

29 

30# ----------------------------- 

31# Imports for other modules -- 

32# ----------------------------- 

33from .quantumGraphExecutor import QuantumGraphExecutor 

34from lsst.base import disableImplicitThreading 

35 

36_LOG = logging.getLogger(__name__.partition(".")[2]) 

37 

38 

39class MPGraphExecutorError(Exception): 

40 """Exception class for errors raised by MPGraphExecutor. 

41 """ 

42 pass 

43 

44 

45class MPGraphExecutor(QuantumGraphExecutor): 

46 """Implementation of QuantumGraphExecutor using same-host multiprocess 

47 execution of Quanta. 

48 

49 Parameters 

50 ---------- 

51 numProc : `int` 

52 Number of processes to use for executing tasks. 

53 timeout : `float` 

54 Time in seconds to wait for tasks to finish. 

55 quantumExecutor : `QuantumExecutor` 

56 Executor for single quantum. For multiprocess-style execution when 

57 ``numProc`` is greater than one this instance must support pickle. 

58 executionGraphFixup : `ExecutionGraphFixup`, optional 

59 Instance used for modification of execution graph. 

60 """ 

61 def __init__(self, numProc, timeout, quantumExecutor, *, executionGraphFixup=None): 

62 self.numProc = numProc 

63 self.timeout = timeout 

64 self.quantumExecutor = quantumExecutor 

65 self.executionGraphFixup = executionGraphFixup 

66 

67 def execute(self, graph, butler): 

68 # Docstring inherited from QuantumGraphExecutor.execute 

69 quantaIter = self._fixupQuanta(graph.traverse()) 

70 if self.numProc > 1: 

71 self._executeQuantaMP(quantaIter, butler) 

72 else: 

73 self._executeQuantaInProcess(quantaIter, butler) 

74 

75 def _fixupQuanta(self, quantaIter): 

76 """Call fixup code to modify execution graph. 

77 

78 Parameters 

79 ---------- 

80 quantaIter : iterable of `~lsst.pipe.base.QuantumIterData` 

81 Quanta as originated from a quantum graph. 

82 

83 Returns 

84 ------- 

85 quantaIter : iterable of `~lsst.pipe.base.QuantumIterData` 

86 Possibly updated set of quanta, properly ordered for execution. 

87 

88 Raises 

89 ------ 

90 MPGraphExecutorError 

91 Raised if execution graph cannot be ordered after modification, 

92 i.e. it has dependency cycles. 

93 """ 

94 if not self.executionGraphFixup: 

95 return quantaIter 

96 

97 _LOG.debug("Call execution graph fixup method") 

98 quantaIter = self.executionGraphFixup.fixupQuanta(quantaIter) 

99 

100 # need it correctly ordered as dependencies may have changed 

101 # after modification, so do topo-sort 

102 updatedQuanta = list(quantaIter) 

103 quanta = [] 

104 ids = set() 

105 _LOG.debug("Re-ordering execution graph") 

106 while updatedQuanta: 

107 # find quantum that has all dependencies resolved already 

108 for i, qdata in enumerate(updatedQuanta): 

109 if ids.issuperset(qdata.dependencies): 

110 _LOG.debug("Found next quanta to execute: %s", qdata) 

111 del updatedQuanta[i] 

112 ids.add(qdata.index) 

113 # we could yield here but I want to detect cycles before 

114 # returning anything from this method 

115 quanta.append(qdata) 

116 break 

117 else: 

118 # means remaining quanta have dependency cycle 

119 raise MPGraphExecutorError( 

120 "Updated execution graph has dependency clycle.") 

121 

122 return quanta 

123 

124 def _executeQuantaInProcess(self, iterable, butler): 

125 """Execute all Quanta in current process. 

126 

127 Parameters 

128 ---------- 

129 iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

130 Sequence if Quanta to execute. It is guaranteed that re-requisites 

131 for a given Quantum will always appear before that Quantum. 

132 butler : `lsst.daf.butler.Butler` 

133 Data butler instance 

134 """ 

135 for qdata in iterable: 

136 _LOG.debug("Executing %s", qdata) 

137 self._executePipelineTask(taskDef=qdata.taskDef, quantum=qdata.quantum, 

138 butler=butler, executor=self.quantumExecutor) 

139 

140 def _executeQuantaMP(self, iterable, butler): 

141 """Execute all Quanta in separate process pool. 

142 

143 Parameters 

144 ---------- 

145 iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

146 Sequence if Quanta to execute. It is guaranteed that re-requisites 

147 for a given Quantum will always appear before that Quantum. 

148 butler : `lsst.daf.butler.Butler` 

149 Data butler instance 

150 """ 

151 

152 disableImplicitThreading() # To prevent thread contention 

153 

154 pool = multiprocessing.Pool(processes=self.numProc, maxtasksperchild=1) 

155 

156 # map quantum id to AsyncResult 

157 results = {} 

158 

159 # Add each Quantum to a pool, wait until it pre-requisites completed. 

160 # TODO: This is not super-efficient as it stops at the first Quantum 

161 # that cannot be executed (yet) and does not check other Quanta. 

162 for qdata in iterable: 

163 

164 # check that task can run in sub-process 

165 taskDef = qdata.taskDef 

166 if not taskDef.taskClass.canMultiprocess: 

167 raise MPGraphExecutorError(f"Task {taskDef.taskName} does not support multiprocessing;" 

168 " use single process") 

169 

170 # Wait for all dependencies 

171 for dep in qdata.dependencies: 

172 # Wait for max. timeout for this result to be ready. 

173 # This can raise on timeout or if remote call raises. 

174 _LOG.debug("Check dependency %s for %s", dep, qdata) 

175 results[dep].get(self.timeout) 

176 _LOG.debug("Result %s is ready", dep) 

177 

178 # Add it to the pool and remember its result 

179 _LOG.debug("Sumbitting %s", qdata) 

180 kwargs = dict(taskDef=taskDef, quantum=qdata.quantum, 

181 butler=butler, executor=self.quantumExecutor) 

182 results[qdata.index] = pool.apply_async(self._executePipelineTask, (), kwargs) 

183 

184 # Everything is submitted, wait until it's complete 

185 _LOG.debug("Wait for all tasks") 

186 for qid, res in results.items(): 

187 if res.ready(): 

188 _LOG.debug("Result %d is ready", qid) 

189 else: 

190 _LOG.debug("Waiting for result %d", qid) 

191 res.get(self.timeout) 

192 

193 @staticmethod 

194 def _executePipelineTask(*, taskDef, quantum, butler, executor): 

195 """Execute PipelineTask on a single data item. 

196 

197 Parameters 

198 ---------- 

199 taskDef : `~lsst.pipe.base.TaskDef` 

200 Task definition structure. 

201 quantum : `~lsst.daf.butler.Quantum` 

202 Quantum for this execution. 

203 butler : `~lsst.daf.butler.Butler` 

204 Data butler instance. 

205 executor : `QuantumExecutor` 

206 Executor for single quantum. 

207 """ 

208 return executor.execute(taskDef, quantum, butler)