Coverage for python/lsst/ctrl/mpexec/mpGraphExecutor.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ['MPGraphExecutor']
24# -------------------------------
25# Imports of standard modules --
26# -------------------------------
27import logging
28import multiprocessing
30# -----------------------------
31# Imports for other modules --
32# -----------------------------
33from .quantumGraphExecutor import QuantumGraphExecutor
34from .singleQuantumExecutor import SingleQuantumExecutor
35from lsst.base import disableImplicitThreading
37_LOG = logging.getLogger(__name__.partition(".")[2])
40class MPGraphExecutorError(Exception):
41 """Exception class for errors raised by MPGraphExecutor.
42 """
43 pass
46class MPGraphExecutor(QuantumGraphExecutor):
47 """Implementation of QuantumGraphExecutor using same-host multiprocess
48 execution of Quanta.
50 Parameters
51 ----------
52 numProc : `int`
53 Number of processes to use for executing tasks.
54 timeout : `float`
55 Time in seconds to wait for tasks to finish.
56 skipExisting : `bool`, optional
57 If True then quanta with all existing outputs are not executed.
58 clobberOutput : `bool`, optional
59 It `True` then override all existing output datasets in an output
60 collection.
61 enableLsstDebug : `bool`, optional
62 Enable debugging with ``lsstDebug`` facility for a task.
63 """
64 def __init__(self, numProc, timeout, skipExisting=False, clobberOutput=False, enableLsstDebug=False):
65 self.numProc = numProc
66 self.timeout = timeout
67 self.skipExisting = skipExisting
68 self.clobberOutput = clobberOutput
69 self.enableLsstDebug = enableLsstDebug
71 def execute(self, graph, butler, taskFactory):
72 # Docstring inherited from QuantumGraphExecutor.execute
73 if self.numProc > 1:
74 self._executeQuantaMP(graph.traverse(), butler, taskFactory)
75 else:
76 self._executeQuantaInProcess(graph.traverse(), butler, taskFactory)
78 def _executeQuantaInProcess(self, iterable, butler, taskFactory):
79 """Execute all Quanta in current process.
81 Parameters
82 ----------
83 iterable : iterable of `~lsst.pipe.base.QuantumIterData`
84 Sequence if Quanta to execute. It is guaranteed that re-requisites
85 for a given Quantum will always appear before that Quantum.
86 butler : `lsst.daf.butler.Butler`
87 Data butler instance
88 taskFactory : `~lsst.pipe.base.TaskFactory`
89 Task factory.
90 """
91 for qdata in iterable:
92 _LOG.debug("Executing %s", qdata)
93 taskDef = qdata.taskDef
94 self._executePipelineTask(taskDef=taskDef, quantum=qdata.quantum, butler=butler,
95 taskFactory=taskFactory, skipExisting=self.skipExisting,
96 clobberOutput=self.clobberOutput,
97 enableLsstDebug=self.enableLsstDebug)
99 def _executeQuantaMP(self, iterable, butler, taskFactory):
100 """Execute all Quanta in separate process pool.
102 Parameters
103 ----------
104 iterable : iterable of `~lsst.pipe.base.QuantumIterData`
105 Sequence if Quanta to execute. It is guaranteed that re-requisites
106 for a given Quantum will always appear before that Quantum.
107 butler : `lsst.daf.butler.Butler`
108 Data butler instance
109 taskFactory : `~lsst.pipe.base.TaskFactory`
110 Task factory.
111 """
113 disableImplicitThreading() # To prevent thread contention
115 pool = multiprocessing.Pool(processes=self.numProc, maxtasksperchild=1)
117 # map quantum id to AsyncResult
118 results = {}
120 # Add each Quantum to a pool, wait until it pre-requisites completed.
121 # TODO: This is not super-efficient as it stops at the first Quantum
122 # that cannot be executed (yet) and does not check other Quanta.
123 for qdata in iterable:
125 # check that task can run in sub-process
126 taskDef = qdata.taskDef
127 if not taskDef.taskClass.canMultiprocess:
128 raise MPGraphExecutorError(f"Task {taskDef.taskName} does not support multiprocessing;"
129 " use single process")
131 # Wait for all dependencies
132 for dep in qdata.dependencies:
133 # Wait for max. timeout for this result to be ready.
134 # This can raise on timeout or if remote call raises.
135 _LOG.debug("Check dependency %s for %s", dep, qdata)
136 results[dep].get(self.timeout)
137 _LOG.debug("Result %s is ready", dep)
139 # Add it to the pool and remember its result
140 _LOG.debug("Sumbitting %s", qdata)
141 kwargs = dict(taskDef=taskDef, quantum=qdata.quantum, butler=butler, taskFactory=taskFactory,
142 skipExisting=self.skipExisting, clobberOutput=self.clobberOutput,
143 enableLsstDebug=self.enableLsstDebug)
144 results[qdata.index] = pool.apply_async(self._executePipelineTask, (), kwargs)
146 # Everything is submitted, wait until it's complete
147 _LOG.debug("Wait for all tasks")
148 for qid, res in results.items():
149 if res.ready():
150 _LOG.debug("Result %d is ready", qid)
151 else:
152 _LOG.debug("Waiting for result %d", qid)
153 res.get(self.timeout)
155 @staticmethod
156 def _executePipelineTask(*, taskDef, quantum, butler, taskFactory, skipExisting,
157 clobberOutput, enableLsstDebug):
158 """Execute PipelineTask on a single data item.
160 Parameters
161 ----------
162 taskDef : `~lsst.pipe.base.TaskDef`
163 Task definition structure.
164 quantum : `~lsst.daf.butler.Quantum`
165 Quantum for this execution.
166 butler : `~lsst.daf.butler.Butler`
167 Data butler instance.
168 taskFactory : `~lsst.pipe.base.TaskFactory`
169 Task factory.
170 skipExisting : `bool`
171 If True then quanta with all existing outputs are not executed.
172 clobberOutput : `bool`, optional
173 It `True` then override all existing output datasets in an output
174 collection.
175 enableLsstDebug : `bool`, optional
176 Enable debugging with ``lsstDebug`` facility for a task.
177 """
178 executor = SingleQuantumExecutor(butler, taskFactory, skipExisting, clobberOutput, enableLsstDebug)
179 return executor.execute(taskDef, quantum)