Coverage for python/lsst/ctrl/mpexec/singleQuantumExecutor.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ['SingleQuantumExecutor']
24# -------------------------------
25# Imports of standard modules --
26# -------------------------------
27import logging
28from itertools import chain
30# -----------------------------
31# Imports for other modules --
32# -----------------------------
33from .quantumGraphExecutor import QuantumExecutor
34from lsst.log import Log
35from lsst.pipe.base import ButlerQuantumContext
37# ----------------------------------
38# Local non-exported definitions --
39# ----------------------------------
41_LOG = logging.getLogger(__name__.partition(".")[2])
44class SingleQuantumExecutor(QuantumExecutor):
45 """Executor class which runs one Quantum at a time.
47 Parameters
48 ----------
49 butler : `~lsst.daf.butler.Butler`
50 Data butler.
51 taskFactory : `~lsst.pipe.base.TaskFactory`
52 Instance of a task factory.
53 skipExisting : `bool`, optional
54 If True then quanta with all existing outputs are not executed.
55 enableLsstDebug : `bool`, optional
56 Enable debugging with ``lsstDebug`` facility for a task.
57 """
58 def __init__(self, taskFactory, skipExisting=False, enableLsstDebug=False):
59 self.taskFactory = taskFactory
60 self.skipExisting = skipExisting
61 self.enableLsstDebug = enableLsstDebug
63 def execute(self, taskDef, quantum, butler):
64 # Docstring inherited from QuantumExecutor.execute
65 taskClass, config = taskDef.taskClass, taskDef.config
66 self.setupLogging(taskClass, config, quantum)
67 if self.skipExisting and self.quantumOutputsExist(quantum, butler):
68 _LOG.info("Quantum execution skipped due to existing outputs, "
69 f"task={taskClass.__name__} dataId={quantum.dataId}.")
70 return
71 self.updateQuantumInputs(quantum, butler)
73 # enable lsstDebug debugging
74 if self.enableLsstDebug:
75 try:
76 _LOG.debug("Will try to import debug.py")
77 import debug # noqa:F401
78 except ImportError:
79 _LOG.warn("No 'debug' module found.")
81 task = self.makeTask(taskClass, config, butler)
82 self.runQuantum(task, quantum, taskDef, butler)
84 def setupLogging(self, taskClass, config, quantum):
85 """Configure logging system for execution of this task.
87 Ths method can setup logging to attach task- or
88 quantum-specific information to log messages. Potentially this can
89 take into accout some info from task configuration as well.
91 Parameters
92 ----------
93 taskClass : `type`
94 Sub-class of `~lsst.pipe.base.PipelineTask`.
95 config : `~lsst.pipe.base.PipelineTaskConfig`
96 Configuration object for this task
97 quantum : `~lsst.daf.butler.Quantum`
98 Single Quantum instance.
99 """
100 # include input dataIds into MDC
101 dataIds = set(ref.dataId for ref in chain.from_iterable(quantum.predictedInputs.values()))
102 if dataIds:
103 if len(dataIds) == 1:
104 Log.MDC("LABEL", str(dataIds.pop()))
105 else:
106 Log.MDC("LABEL", '[' + ', '.join([str(dataId) for dataId in dataIds]) + ']')
108 def quantumOutputsExist(self, quantum, butler):
109 """Decide whether this quantum needs to be executed.
111 Parameters
112 ----------
113 quantum : `~lsst.daf.butler.Quantum`
114 Quantum to check for existing outputs
115 butler : `~lsst.daf.butler.Butler`
116 Data butler.
118 Returns
119 -------
120 exist : `bool`
121 True if all quantum's outputs exist in a collection, False
122 otherwise.
124 Raises
125 ------
126 RuntimeError
127 Raised if some outputs exist and some not.
128 """
129 collection = butler.run
130 registry = butler.registry
132 existingRefs = []
133 missingRefs = []
134 for datasetRefs in quantum.outputs.values():
135 for datasetRef in datasetRefs:
136 ref = registry.findDataset(datasetRef.datasetType, datasetRef.dataId,
137 collections=butler.run)
138 if ref is None:
139 missingRefs.append(datasetRefs)
140 else:
141 existingRefs.append(datasetRefs)
142 if existingRefs and missingRefs:
143 # some outputs exist and same not, can't do a thing with that
144 raise RuntimeError(f"Registry inconsistency while checking for existing outputs:"
145 f" collection={collection} existingRefs={existingRefs}"
146 f" missingRefs={missingRefs}")
147 else:
148 return bool(existingRefs)
150 def makeTask(self, taskClass, config, butler):
151 """Make new task instance.
153 Parameters
154 ----------
155 taskClass : `type`
156 Sub-class of `~lsst.pipe.base.PipelineTask`.
157 config : `~lsst.pipe.base.PipelineTaskConfig`
158 Configuration object for this task
160 Returns
161 -------
162 task : `~lsst.pipe.base.PipelineTask`
163 Instance of ``taskClass`` type.
164 butler : `~lsst.daf.butler.Butler`
165 Data butler.
166 """
167 # call task factory for that
168 return self.taskFactory.makeTask(taskClass, config, None, butler)
170 def updateQuantumInputs(self, quantum, butler):
171 """Update quantum with extra information.
173 Some methods may require input DatasetRefs to have non-None
174 ``dataset_id``, but in case of intermediate dataset it may not be
175 filled during QuantumGraph construction. This method will retrieve
176 missing info from registry.
178 Parameters
179 ----------
180 quantum : `~lsst.daf.butler.Quantum`
181 Single Quantum instance.
182 butler : `~lsst.daf.butler.Butler`
183 Data butler.
184 """
185 for refsForDatasetType in quantum.predictedInputs.values():
186 newRefsForDatasetType = []
187 for ref in refsForDatasetType:
188 if ref.id is None:
189 resolvedRef = butler.registry.findDataset(ref.datasetType, ref.dataId,
190 collections=butler.collections)
191 if resolvedRef is None:
192 raise ValueError(
193 f"Cannot find {ref.datasetType.name} with id {ref.dataId} "
194 f"in collections {butler.collections}."
195 )
196 newRefsForDatasetType.append(resolvedRef)
197 _LOG.debug("Updating dataset ID for %s", ref)
198 else:
199 newRefsForDatasetType.append(ref)
200 refsForDatasetType[:] = newRefsForDatasetType
202 def runQuantum(self, task, quantum, taskDef, butler):
203 """Execute task on a single quantum.
205 Parameters
206 ----------
207 task : `~lsst.pipe.base.PipelineTask`
208 Task object.
209 quantum : `~lsst.daf.butler.Quantum`
210 Single Quantum instance.
211 taskDef : `~lsst.pipe.base.TaskDef`
212 Task definition structure.
213 butler : `~lsst.daf.butler.Butler`
214 Data butler.
215 """
216 # Create a butler that operates in the context of a quantum
217 butlerQC = ButlerQuantumContext(butler, quantum)
219 # Get the input and output references for the task
220 connectionInstance = task.config.connections.ConnectionsClass(config=task.config)
221 inputRefs, outputRefs = connectionInstance.buildDatasetRefs(quantum)
222 # Call task runQuantum() method. Any exception thrown by the task
223 # propagates to caller.
224 task.runQuantum(butlerQC, inputRefs, outputRefs)
226 if taskDef.metadataDatasetName is not None:
227 # DatasetRef has to be in the Quantum outputs, can lookup by name
228 try:
229 ref = quantum.outputs[taskDef.metadataDatasetName]
230 except LookupError as exc:
231 raise LookupError(
232 f"Quantum outputs is missing metadata dataset type {taskDef.metadataDatasetName},"
233 f" it could happen due to inconsistent options between Quantum generation"
234 f" and execution") from exc
235 butlerQC.put(task.metadata, ref[0])