Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['SingleQuantumExecutor'] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28from itertools import chain 

29 

30# ----------------------------- 

31# Imports for other modules -- 

32# ----------------------------- 

33from .quantumGraphExecutor import QuantumExecutor 

34from lsst.log import Log 

35from lsst.pipe.base import ButlerQuantumContext 

36 

37# ---------------------------------- 

38# Local non-exported definitions -- 

39# ---------------------------------- 

40 

41_LOG = logging.getLogger(__name__.partition(".")[2]) 

42 

43 

44class SingleQuantumExecutor(QuantumExecutor): 

45 """Executor class which runs one Quantum at a time. 

46 

47 Parameters 

48 ---------- 

49 butler : `~lsst.daf.butler.Butler` 

50 Data butler. 

51 taskFactory : `~lsst.pipe.base.TaskFactory` 

52 Instance of a task factory. 

53 skipExisting : `bool`, optional 

54 If True then quanta with all existing outputs are not executed. 

55 enableLsstDebug : `bool`, optional 

56 Enable debugging with ``lsstDebug`` facility for a task. 

57 """ 

58 def __init__(self, taskFactory, skipExisting=False, enableLsstDebug=False): 

59 self.taskFactory = taskFactory 

60 self.skipExisting = skipExisting 

61 self.enableLsstDebug = enableLsstDebug 

62 

63 def execute(self, taskDef, quantum, butler): 

64 # Docstring inherited from QuantumExecutor.execute 

65 taskClass, config = taskDef.taskClass, taskDef.config 

66 self.setupLogging(taskClass, config, quantum) 

67 if self.skipExisting and self.quantumOutputsExist(quantum, butler): 

68 _LOG.info("Quantum execution skipped due to existing outputs, " 

69 f"task={taskClass.__name__} dataId={quantum.dataId}.") 

70 return 

71 self.updateQuantumInputs(quantum, butler) 

72 

73 # enable lsstDebug debugging 

74 if self.enableLsstDebug: 

75 try: 

76 _LOG.debug("Will try to import debug.py") 

77 import debug # noqa:F401 

78 except ImportError: 

79 _LOG.warn("No 'debug' module found.") 

80 

81 task = self.makeTask(taskClass, config, butler) 

82 self.runQuantum(task, quantum, taskDef, butler) 

83 

84 def setupLogging(self, taskClass, config, quantum): 

85 """Configure logging system for execution of this task. 

86 

87 Ths method can setup logging to attach task- or 

88 quantum-specific information to log messages. Potentially this can 

89 take into accout some info from task configuration as well. 

90 

91 Parameters 

92 ---------- 

93 taskClass : `type` 

94 Sub-class of `~lsst.pipe.base.PipelineTask`. 

95 config : `~lsst.pipe.base.PipelineTaskConfig` 

96 Configuration object for this task 

97 quantum : `~lsst.daf.butler.Quantum` 

98 Single Quantum instance. 

99 """ 

100 # include input dataIds into MDC 

101 dataIds = set(ref.dataId for ref in chain.from_iterable(quantum.predictedInputs.values())) 

102 if dataIds: 

103 if len(dataIds) == 1: 

104 Log.MDC("LABEL", str(dataIds.pop())) 

105 else: 

106 Log.MDC("LABEL", '[' + ', '.join([str(dataId) for dataId in dataIds]) + ']') 

107 

108 def quantumOutputsExist(self, quantum, butler): 

109 """Decide whether this quantum needs to be executed. 

110 

111 Parameters 

112 ---------- 

113 quantum : `~lsst.daf.butler.Quantum` 

114 Quantum to check for existing outputs 

115 butler : `~lsst.daf.butler.Butler` 

116 Data butler. 

117 

118 Returns 

119 ------- 

120 exist : `bool` 

121 True if all quantum's outputs exist in a collection, False 

122 otherwise. 

123 

124 Raises 

125 ------ 

126 RuntimeError 

127 Raised if some outputs exist and some not. 

128 """ 

129 collection = butler.run 

130 registry = butler.registry 

131 

132 existingRefs = [] 

133 missingRefs = [] 

134 for datasetRefs in quantum.outputs.values(): 

135 for datasetRef in datasetRefs: 

136 ref = registry.findDataset(datasetRef.datasetType, datasetRef.dataId, 

137 collections=butler.run) 

138 if ref is None: 

139 missingRefs.append(datasetRefs) 

140 else: 

141 existingRefs.append(datasetRefs) 

142 if existingRefs and missingRefs: 

143 # some outputs exist and same not, can't do a thing with that 

144 raise RuntimeError(f"Registry inconsistency while checking for existing outputs:" 

145 f" collection={collection} existingRefs={existingRefs}" 

146 f" missingRefs={missingRefs}") 

147 else: 

148 return bool(existingRefs) 

149 

150 def makeTask(self, taskClass, config, butler): 

151 """Make new task instance. 

152 

153 Parameters 

154 ---------- 

155 taskClass : `type` 

156 Sub-class of `~lsst.pipe.base.PipelineTask`. 

157 config : `~lsst.pipe.base.PipelineTaskConfig` 

158 Configuration object for this task 

159 

160 Returns 

161 ------- 

162 task : `~lsst.pipe.base.PipelineTask` 

163 Instance of ``taskClass`` type. 

164 butler : `~lsst.daf.butler.Butler` 

165 Data butler. 

166 """ 

167 # call task factory for that 

168 return self.taskFactory.makeTask(taskClass, config, None, butler) 

169 

170 def updateQuantumInputs(self, quantum, butler): 

171 """Update quantum with extra information. 

172 

173 Some methods may require input DatasetRefs to have non-None 

174 ``dataset_id``, but in case of intermediate dataset it may not be 

175 filled during QuantumGraph construction. This method will retrieve 

176 missing info from registry. 

177 

178 Parameters 

179 ---------- 

180 quantum : `~lsst.daf.butler.Quantum` 

181 Single Quantum instance. 

182 butler : `~lsst.daf.butler.Butler` 

183 Data butler. 

184 """ 

185 for refsForDatasetType in quantum.predictedInputs.values(): 

186 newRefsForDatasetType = [] 

187 for ref in refsForDatasetType: 

188 if ref.id is None: 

189 resolvedRef = butler.registry.findDataset(ref.datasetType, ref.dataId, 

190 collections=butler.collections) 

191 if resolvedRef is None: 

192 raise ValueError( 

193 f"Cannot find {ref.datasetType.name} with id {ref.dataId} " 

194 f"in collections {butler.collections}." 

195 ) 

196 newRefsForDatasetType.append(resolvedRef) 

197 _LOG.debug("Updating dataset ID for %s", ref) 

198 else: 

199 newRefsForDatasetType.append(ref) 

200 refsForDatasetType[:] = newRefsForDatasetType 

201 

202 def runQuantum(self, task, quantum, taskDef, butler): 

203 """Execute task on a single quantum. 

204 

205 Parameters 

206 ---------- 

207 task : `~lsst.pipe.base.PipelineTask` 

208 Task object. 

209 quantum : `~lsst.daf.butler.Quantum` 

210 Single Quantum instance. 

211 taskDef : `~lsst.pipe.base.TaskDef` 

212 Task definition structure. 

213 butler : `~lsst.daf.butler.Butler` 

214 Data butler. 

215 """ 

216 # Create a butler that operates in the context of a quantum 

217 butlerQC = ButlerQuantumContext(butler, quantum) 

218 

219 # Get the input and output references for the task 

220 connectionInstance = task.config.connections.ConnectionsClass(config=task.config) 

221 inputRefs, outputRefs = connectionInstance.buildDatasetRefs(quantum) 

222 # Call task runQuantum() method. Any exception thrown by the task 

223 # propagates to caller. 

224 task.runQuantum(butlerQC, inputRefs, outputRefs) 

225 

226 if taskDef.metadataDatasetName is not None: 

227 # DatasetRef has to be in the Quantum outputs, can lookup by name 

228 try: 

229 ref = quantum.outputs[taskDef.metadataDatasetName] 

230 except LookupError as exc: 

231 raise LookupError( 

232 f"Quantum outputs is missing metadata dataset type {taskDef.metadataDatasetName}," 

233 f" it could happen due to inconsistent options between Quantum generation" 

234 f" and execution") from exc 

235 butlerQC.put(task.metadata, ref[0])