Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['MPGraphExecutor'] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28import multiprocessing 

29 

30# ----------------------------- 

31# Imports for other modules -- 

32# ----------------------------- 

33from .quantumGraphExecutor import QuantumGraphExecutor 

34from .singleQuantumExecutor import SingleQuantumExecutor 

35from lsst.base import disableImplicitThreading 

36 

37_LOG = logging.getLogger(__name__.partition(".")[2]) 

38 

39 

40class MPGraphExecutorError(Exception): 

41 """Exception class for errors raised by MPGraphExecutor. 

42 """ 

43 pass 

44 

45 

46class MPGraphExecutor(QuantumGraphExecutor): 

47 """Implementation of QuantumGraphExecutor using same-host multiprocess 

48 execution of Quanta. 

49 

50 Parameters 

51 ---------- 

52 numProc : `int` 

53 Number of processes to use for executing tasks. 

54 timeout : `float` 

55 Time in seconds to wait for tasks to finish. 

56 skipExisting : `bool`, optional 

57 If True then quanta with all existing outputs are not executed. 

58 clobberOutput : `bool`, optional 

59 It `True` then override all existing output datasets in an output 

60 collection. 

61 enableLsstDebug : `bool`, optional 

62 Enable debugging with ``lsstDebug`` facility for a task. 

63 """ 

64 def __init__(self, numProc, timeout, skipExisting=False, clobberOutput=False, enableLsstDebug=False): 

65 self.numProc = numProc 

66 self.timeout = timeout 

67 self.skipExisting = skipExisting 

68 self.clobberOutput = clobberOutput 

69 self.enableLsstDebug = enableLsstDebug 

70 

71 def execute(self, graph, butler, taskFactory): 

72 # Docstring inherited from QuantumGraphExecutor.execute 

73 if self.numProc > 1: 

74 self._executeQuantaMP(graph.traverse(), butler, taskFactory) 

75 else: 

76 self._executeQuantaInProcess(graph.traverse(), butler, taskFactory) 

77 

78 def _executeQuantaInProcess(self, iterable, butler, taskFactory): 

79 """Execute all Quanta in current process. 

80 

81 Parameters 

82 ---------- 

83 iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

84 Sequence if Quanta to execute. It is guaranteed that re-requisites 

85 for a given Quantum will always appear before that Quantum. 

86 butler : `lsst.daf.butler.Butler` 

87 Data butler instance 

88 taskFactory : `~lsst.pipe.base.TaskFactory` 

89 Task factory. 

90 """ 

91 for qdata in iterable: 

92 _LOG.debug("Executing %s", qdata) 

93 taskDef = qdata.taskDef 

94 self._executePipelineTask(taskDef=taskDef, quantum=qdata.quantum, butler=butler, 

95 taskFactory=taskFactory, skipExisting=self.skipExisting, 

96 clobberOutput=self.clobberOutput, 

97 enableLsstDebug=self.enableLsstDebug) 

98 

99 def _executeQuantaMP(self, iterable, butler, taskFactory): 

100 """Execute all Quanta in separate process pool. 

101 

102 Parameters 

103 ---------- 

104 iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

105 Sequence if Quanta to execute. It is guaranteed that re-requisites 

106 for a given Quantum will always appear before that Quantum. 

107 butler : `lsst.daf.butler.Butler` 

108 Data butler instance 

109 taskFactory : `~lsst.pipe.base.TaskFactory` 

110 Task factory. 

111 """ 

112 

113 disableImplicitThreading() # To prevent thread contention 

114 

115 pool = multiprocessing.Pool(processes=self.numProc, maxtasksperchild=1) 

116 

117 # map quantum id to AsyncResult 

118 results = {} 

119 

120 # Add each Quantum to a pool, wait until it pre-requisites completed. 

121 # TODO: This is not super-efficient as it stops at the first Quantum 

122 # that cannot be executed (yet) and does not check other Quanta. 

123 for qdata in iterable: 

124 

125 # check that task can run in sub-process 

126 taskDef = qdata.taskDef 

127 if not taskDef.taskClass.canMultiprocess: 

128 raise MPGraphExecutorError(f"Task {taskDef.taskName} does not support multiprocessing;" 

129 " use single process") 

130 

131 # Wait for all dependencies 

132 for dep in qdata.dependencies: 

133 # Wait for max. timeout for this result to be ready. 

134 # This can raise on timeout or if remote call raises. 

135 _LOG.debug("Check dependency %s for %s", dep, qdata) 

136 results[dep].get(self.timeout) 

137 _LOG.debug("Result %s is ready", dep) 

138 

139 # Add it to the pool and remember its result 

140 _LOG.debug("Sumbitting %s", qdata) 

141 kwargs = dict(taskDef=taskDef, quantum=qdata.quantum, butler=butler, taskFactory=taskFactory, 

142 skipExisting=self.skipExisting, clobberOutput=self.clobberOutput, 

143 enableLsstDebug=self.enableLsstDebug) 

144 results[qdata.index] = pool.apply_async(self._executePipelineTask, (), kwargs) 

145 

146 # Everything is submitted, wait until it's complete 

147 _LOG.debug("Wait for all tasks") 

148 for qid, res in results.items(): 

149 if res.ready(): 

150 _LOG.debug("Result %d is ready", qid) 

151 else: 

152 _LOG.debug("Waiting for result %d", qid) 

153 res.get(self.timeout) 

154 

155 @staticmethod 

156 def _executePipelineTask(*, taskDef, quantum, butler, taskFactory, skipExisting, 

157 clobberOutput, enableLsstDebug): 

158 """Execute PipelineTask on a single data item. 

159 

160 Parameters 

161 ---------- 

162 taskDef : `~lsst.pipe.base.TaskDef` 

163 Task definition structure. 

164 quantum : `~lsst.daf.butler.Quantum` 

165 Quantum for this execution. 

166 butler : `~lsst.daf.butler.Butler` 

167 Data butler instance. 

168 taskFactory : `~lsst.pipe.base.TaskFactory` 

169 Task factory. 

170 skipExisting : `bool` 

171 If True then quanta with all existing outputs are not executed. 

172 clobberOutput : `bool`, optional 

173 It `True` then override all existing output datasets in an output 

174 collection. 

175 enableLsstDebug : `bool`, optional 

176 Enable debugging with ``lsstDebug`` facility for a task. 

177 """ 

178 executor = SingleQuantumExecutor(butler, taskFactory, skipExisting, clobberOutput, enableLsstDebug) 

179 return executor.execute(taskDef, quantum)