Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['MPGraphExecutor'] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28import multiprocessing 

29 

30# ----------------------------- 

31# Imports for other modules -- 

32# ----------------------------- 

33from .quantumGraphExecutor import QuantumGraphExecutor 

34from .singleQuantumExecutor import SingleQuantumExecutor 

35from lsst.base import disableImplicitThreading 

36 

37_LOG = logging.getLogger(__name__.partition(".")[2]) 

38 

39 

40class MPGraphExecutorError(Exception): 

41 """Exception class for errors raised by MPGraphExecutor. 

42 """ 

43 pass 

44 

45 

46class MPGraphExecutor(QuantumGraphExecutor): 

47 """Implementation of QuantumGraphExecutor using same-host multiprocess 

48 execution of Quanta. 

49 

50 Parameters 

51 ---------- 

52 numProc : `int` 

53 Number of processes to use for executing tasks. 

54 timeout : `float` 

55 Time in seconds to wait for tasks to finish. 

56 skipExisting : `bool`, optional 

57 If True then quanta with all existing outputs are not executed. 

58 enableLsstDebug : `bool`, optional 

59 Enable debugging with ``lsstDebug`` facility for a task. 

60 """ 

61 def __init__(self, numProc, timeout, skipExisting=False, enableLsstDebug=False): 

62 self.numProc = numProc 

63 self.timeout = timeout 

64 self.skipExisting = skipExisting 

65 self.enableLsstDebug = enableLsstDebug 

66 

67 def execute(self, graph, butler, taskFactory): 

68 # Docstring inherited from QuantumGraphExecutor.execute 

69 if self.numProc > 1: 

70 self._executeQuantaMP(graph.traverse(), butler, taskFactory) 

71 else: 

72 self._executeQuantaInProcess(graph.traverse(), butler, taskFactory) 

73 

74 def _executeQuantaInProcess(self, iterable, butler, taskFactory): 

75 """Execute all Quanta in current process. 

76 

77 Parameters 

78 ---------- 

79 iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

80 Sequence if Quanta to execute. It is guaranteed that re-requisites 

81 for a given Quantum will always appear before that Quantum. 

82 butler : `lsst.daf.butler.Butler` 

83 Data butler instance 

84 taskFactory : `~lsst.pipe.base.TaskFactory` 

85 Task factory. 

86 """ 

87 for qdata in iterable: 

88 _LOG.debug("Executing %s", qdata) 

89 taskDef = qdata.taskDef 

90 self._executePipelineTask(taskDef=taskDef, quantum=qdata.quantum, butler=butler, 

91 taskFactory=taskFactory, skipExisting=self.skipExisting, 

92 enableLsstDebug=self.enableLsstDebug) 

93 

94 def _executeQuantaMP(self, iterable, butler, taskFactory): 

95 """Execute all Quanta in separate process pool. 

96 

97 Parameters 

98 ---------- 

99 iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

100 Sequence if Quanta to execute. It is guaranteed that re-requisites 

101 for a given Quantum will always appear before that Quantum. 

102 butler : `lsst.daf.butler.Butler` 

103 Data butler instance 

104 taskFactory : `~lsst.pipe.base.TaskFactory` 

105 Task factory. 

106 """ 

107 

108 disableImplicitThreading() # To prevent thread contention 

109 

110 pool = multiprocessing.Pool(processes=self.numProc, maxtasksperchild=1) 

111 

112 # map quantum id to AsyncResult 

113 results = {} 

114 

115 # Add each Quantum to a pool, wait until it pre-requisites completed. 

116 # TODO: This is not super-efficient as it stops at the first Quantum 

117 # that cannot be executed (yet) and does not check other Quanta. 

118 for qdata in iterable: 

119 

120 # check that task can run in sub-process 

121 taskDef = qdata.taskDef 

122 if not taskDef.taskClass.canMultiprocess: 

123 raise MPGraphExecutorError(f"Task {taskDef.taskName} does not support multiprocessing;" 

124 " use single process") 

125 

126 # Wait for all dependencies 

127 for dep in qdata.dependencies: 

128 # Wait for max. timeout for this result to be ready. 

129 # This can raise on timeout or if remote call raises. 

130 _LOG.debug("Check dependency %s for %s", dep, qdata) 

131 results[dep].get(self.timeout) 

132 _LOG.debug("Result %s is ready", dep) 

133 

134 # Add it to the pool and remember its result 

135 _LOG.debug("Sumbitting %s", qdata) 

136 kwargs = dict(taskDef=taskDef, quantum=qdata.quantum, butler=butler, taskFactory=taskFactory, 

137 skipExisting=self.skipExisting, enableLsstDebug=self.enableLsstDebug) 

138 results[qdata.index] = pool.apply_async(self._executePipelineTask, (), kwargs) 

139 

140 # Everything is submitted, wait until it's complete 

141 _LOG.debug("Wait for all tasks") 

142 for qid, res in results.items(): 

143 if res.ready(): 

144 _LOG.debug("Result %d is ready", qid) 

145 else: 

146 _LOG.debug("Waiting for result %d", qid) 

147 res.get(self.timeout) 

148 

149 @staticmethod 

150 def _executePipelineTask(*, taskDef, quantum, butler, taskFactory, skipExisting, enableLsstDebug): 

151 """Execute PipelineTask on a single data item. 

152 

153 Parameters 

154 ---------- 

155 taskDef : `~lsst.pipe.base.TaskDef` 

156 Task definition structure. 

157 quantum : `~lsst.daf.butler.Quantum` 

158 Quantum for this execution. 

159 butler : `~lsst.daf.butler.Butler` 

160 Data butler instance. 

161 taskFactory : `~lsst.pipe.base.TaskFactory` 

162 Task factory. 

163 skipExisting : `bool` 

164 If True then quanta with all existing outputs are not executed. 

165 enableLsstDebug : `bool`, optional 

166 Enable debugging with ``lsstDebug`` facility for a task. 

167 """ 

168 executor = SingleQuantumExecutor(butler, taskFactory, skipExisting, enableLsstDebug) 

169 return executor.execute(taskDef, quantum)