Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

# This file is part of ctrl_mpexec. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

__all__ = ['MPGraphExecutor'] 

 

# ------------------------------- 

# Imports of standard modules -- 

# ------------------------------- 

import logging 

import multiprocessing 

 

# ----------------------------- 

# Imports for other modules -- 

# ----------------------------- 

from .quantumGraphExecutor import QuantumGraphExecutor 

from .singleQuantumExecutor import SingleQuantumExecutor 

from lsst.base import disableImplicitThreading 

 

_LOG = logging.getLogger(__name__.partition(".")[2]) 

 

 

class MPGraphExecutorError(Exception): 

"""Exception class for errors raised by MPGraphExecutor. 

""" 

pass 

 

 

class MPGraphExecutor(QuantumGraphExecutor): 

"""Implementation of QuantumGraphExecutor using same-host multiprocess 

execution of Quanta. 

 

Parameters 

---------- 

numProc : `int` 

Number of processes to use for executing tasks. 

timeout : `float` 

Time in seconds to wait for tasks to finish. 

""" 

def __init__(self, numProc, timeout): 

self.numProc = numProc 

self.timeout = timeout 

 

def execute(self, graph, butler, taskFactory): 

# Docstring inherited from QuantumGraphExecutor.execute 

if self.numProc > 1: 

self._executeQuantaMP(graph.traverse(), butler, taskFactory) 

else: 

self._executeQuantaInProcess(graph.traverse(), butler, taskFactory) 

 

def _executeQuantaInProcess(self, iterable, butler, taskFactory): 

"""Execute all Quanta in current process. 

 

Parameters 

---------- 

iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

Sequence if Quanta to execute. It is guaranteed that re-requisites 

for a given Quantum will always appear before that Quantum. 

butler : `lsst.daf.butler.Butler` 

Data butler instance 

taskFactory : `~lsst.pipe.base.TaskFactory` 

Task factory. 

""" 

for qdata in iterable: 

_LOG.debug("Executing %s", qdata) 

taskDef = qdata.taskDef 

self._executePipelineTask(taskDef.taskClass, taskDef.config, qdata.quantum, butler, taskFactory) 

 

def _executeQuantaMP(self, iterable, butler, taskFactory): 

"""Execute all Quanta in separate process pool. 

 

Parameters 

---------- 

iterable : iterable of `~lsst.pipe.base.QuantumIterData` 

Sequence if Quanta to execute. It is guaranteed that re-requisites 

for a given Quantum will always appear before that Quantum. 

butler : `lsst.daf.butler.Butler` 

Data butler instance 

taskFactory : `~lsst.pipe.base.TaskFactory` 

Task factory. 

""" 

 

disableImplicitThreading() # To prevent thread contention 

 

pool = multiprocessing.Pool(processes=self.numProc, maxtasksperchild=1) 

 

# map quantum id to AsyncResult 

results = {} 

 

# Add each Quantum to a pool, wait until it pre-requisites completed. 

# TODO: This is not super-efficient as it stops at the first Quantum 

# that cannot be executed (yet) and does not check other Quanta. 

for qdata in iterable: 

 

# check that task can run in sub-process 

taskDef = qdata.taskDef 

if not taskDef.taskClass.canMultiprocess: 

raise MPGraphExecutorError(f"Task {taskDef.taskName} does not support multiprocessing;" 

" use single process") 

 

# Wait for all pre-reqs 

for preReq in qdata.prerequisites: 

# Wait for max. timeout for this result to be ready. 

# This can raise on timeout or if remote call raises. 

_LOG.debug("Check pre-req %s for %s", preReq, qdata) 

results[preReq].get(self.timeout) 

_LOG.debug("Result %s is ready", preReq) 

 

# Add it to the pool and remember its result 

_LOG.debug("Sumbitting %s", qdata) 

args = (taskDef.taskClass, taskDef.config, qdata.quantum, butler, taskFactory) 

results[qdata.quantumId] = pool.apply_async(self._executePipelineTask, args) 

 

# Everything is submitted, wait until it's complete 

_LOG.debug("Wait for all tasks") 

for qid, res in results.items(): 

if res.ready(): 

_LOG.debug("Result %d is ready", qid) 

else: 

_LOG.debug("Waiting for result %d", qid) 

res.get(self.timeout) 

 

@staticmethod 

def _executePipelineTask(taskClass, config, quantum, butler, taskFactory): 

"""Execute super-task on a single data item. 

 

Parameters 

---------- 

taskClass : `type` 

Sub-class of PipelineTask. 

config : `~lsst.pipe.base.PipelineTaskConfig` 

Task configuration. 

quantum : `~lsst.daf.butler.Quantum` 

Quantum for this execution. 

butler : `~lsst.daf.butler.Butler` 

Data butler instance. 

taskFactory : `~lsst.pipe.base.TaskFactory` 

Task factory. 

""" 

executor = SingleQuantumExecutor(butler, taskFactory) 

return executor.execute(taskClass, config, quantum)