Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Quantum",) 

25 

26from typing import ( 

27 Iterable, 

28 List, 

29 Mapping, 

30 Optional, 

31 Type, 

32 TYPE_CHECKING, 

33 Union, 

34) 

35 

36import astropy.time 

37 

38from lsst.utils import doImport 

39 

40from .named import NamedKeyDict 

41 

42if TYPE_CHECKING: 42 ↛ 43line 42 didn't jump to line 43, because the condition on line 42 was never true

43 from .dimensions import DataCoordinate 

44 from .datasets import DatasetRef, DatasetType 

45 

46 

47class Quantum: 

48 """A discrete unit of work that may depend on one or more datasets and 

49 produces one or more datasets. 

50 

51 Most Quanta will be executions of a particular ``PipelineTask``’s 

52 ``runQuantum`` method, but they can also be used to represent discrete 

53 units of work performed manually by human operators or other software 

54 agents. 

55 

56 Parameters 

57 ---------- 

58 taskName : `str`, optional 

59 Fully-qualified name of the Task class that executed or will execute 

60 this Quantum. If not provided, ``taskClass`` must be. 

61 taskClass : `type`, optional 

62 The Task class that executed or will execute this Quantum. If not 

63 provided, ``taskName`` must be. Overrides ``taskName`` if both are 

64 provided. 

65 dataId : `DataId`, optional 

66 The dimension values that identify this `Quantum`. 

67 run : `str`, optional 

68 The name of the run this Quantum is a part of. 

69 initInputs : collection of `DatasetRef`, optional 

70 Datasets that are needed to construct an instance of the Task. May 

71 be a flat iterable of `DatasetRef` instances or a mapping from 

72 `DatasetType` to `DatasetRef`. 

73 predictedInputs : `~collections.abc.Mapping`, optional 

74 Inputs identified prior to execution, organized as a mapping from 

75 `DatasetType` to a list of `DatasetRef`. Must be a superset of 

76 ``actualInputs``. 

77 actualInputs : `~collections.abc.Mapping`, optional 

78 Inputs actually used during execution, organized as a mapping from 

79 `DatasetType` to a list of `DatasetRef`. Must be a subset of 

80 ``predictedInputs``. 

81 outputs : `~collections.abc.Mapping`, optional 

82 Outputs from executing this quantum of work, organized as a mapping 

83 from `DatasetType` to a list of `DatasetRef`. 

84 startTime : `astropy.time.Time` 

85 The start time for the quantum. 

86 endTime : `astropy.time.Time` 

87 The end time for the quantum. 

88 host : `str` 

89 The system on this quantum was executed. 

90 id : `int`, optional 

91 Unique integer identifier for this quantum. Usually set to `None` 

92 (default) and assigned by `Registry`. 

93 """ 

94 

95 __slots__ = ("_taskName", "_taskClass", "_dataId", "_run", 

96 "_initInputs", "_predictedInputs", "_actualInputs", "_outputs", 

97 "_id", "_startTime", "_endTime", "_host") 

98 

99 def __init__(self, *, taskName: Optional[str] = None, 

100 taskClass: Optional[Type] = None, 

101 dataId: Optional[DataCoordinate] = None, 

102 run: Optional[str] = None, 

103 initInputs: Optional[Union[Mapping[DatasetType, DatasetRef], Iterable[DatasetRef]]] = None, 

104 predictedInputs: Optional[Mapping[DatasetType, List[DatasetRef]]] = None, 

105 actualInputs: Optional[Mapping[DatasetType, List[DatasetRef]]] = None, 

106 outputs: Optional[Mapping[DatasetType, List[DatasetRef]]] = None, 

107 startTime: Optional[astropy.time.Time] = None, 

108 endTime: Optional[astropy.time.Time] = None, 

109 host: Optional[str] = None, 

110 id: Optional[int] = None): 

111 if taskClass is not None: 

112 taskName = f"{taskClass.__module__}.{taskClass.__name__}" 

113 self._taskName = taskName 

114 self._taskClass = taskClass 

115 self._run = run 

116 self._dataId = dataId 

117 if initInputs is None: 

118 initInputs = {} 

119 elif not isinstance(initInputs, Mapping): 

120 initInputs = {ref.datasetType: ref for ref in initInputs} 

121 if predictedInputs is None: 

122 predictedInputs = {} 

123 if actualInputs is None: 

124 actualInputs = {} 

125 if outputs is None: 

126 outputs = {} 

127 self._initInputs: NamedKeyDict[DatasetType, DatasetRef] = NamedKeyDict(initInputs) 

128 self._predictedInputs: NamedKeyDict[DatasetType, List[DatasetRef]] = NamedKeyDict(predictedInputs) 

129 self._actualInputs: NamedKeyDict[DatasetType, List[DatasetRef]] = NamedKeyDict(actualInputs) 

130 self._outputs: NamedKeyDict[DatasetType, List[DatasetRef]] = NamedKeyDict(outputs) 

131 self._id = id 

132 self._startTime = startTime 

133 self._endTime = endTime 

134 self._host = host 

135 

136 @property 

137 def taskClass(self) -> Optional[Type]: 

138 """Task class associated with this `Quantum` (`type`). 

139 """ 

140 if self._taskClass is None: 

141 self._taskClass = doImport(self._taskName) 

142 return self._taskClass 

143 

144 @property 

145 def taskName(self) -> Optional[str]: 

146 """Fully-qualified name of the task associated with `Quantum` (`str`). 

147 """ 

148 return self._taskName 

149 

150 @property 

151 def run(self) -> Optional[str]: 

152 """The name of the run this Quantum is a part of (`str`). 

153 """ 

154 return self._run 

155 

156 @property 

157 def dataId(self) -> Optional[DataCoordinate]: 

158 """The dimension values of the unit of processing (`DataId`). 

159 """ 

160 return self._dataId 

161 

162 @property 

163 def initInputs(self) -> NamedKeyDict[DatasetType, DatasetRef]: 

164 """A mapping of datasets used to construct the Task, 

165 with `DatasetType` instances as keys (names can also be used for 

166 lookups) and `DatasetRef` instances as values. 

167 """ 

168 return self._initInputs 

169 

170 @property 

171 def predictedInputs(self) -> NamedKeyDict[DatasetType, List[DatasetRef]]: 

172 """A mapping of input datasets that were expected to be used, 

173 with `DatasetType` instances as keys (names can also be used for 

174 lookups) and a list of `DatasetRef` instances as values. 

175 

176 Notes 

177 ----- 

178 We cannot use `set` instead of `list` for the nested container because 

179 `DatasetRef` instances cannot be compared reliably when some have 

180 integers IDs and others do not. 

181 """ 

182 return self._predictedInputs 

183 

184 @property 

185 def actualInputs(self) -> NamedKeyDict[DatasetType, List[DatasetRef]]: 

186 """A mapping of input datasets that were actually used, with the same 

187 form as `Quantum.predictedInputs`. 

188 

189 Notes 

190 ----- 

191 We cannot use `set` instead of `list` for the nested container because 

192 `DatasetRef` instances cannot be compared reliably when some have 

193 integers IDs and others do not. 

194 """ 

195 return self._actualInputs 

196 

197 @property 

198 def outputs(self) -> NamedKeyDict[DatasetType, List[DatasetRef]]: 

199 """A mapping of output datasets (to be) generated for this quantum, 

200 with the same form as `predictedInputs`. 

201 

202 Notes 

203 ----- 

204 We cannot use `set` instead of `list` for the nested container because 

205 `DatasetRef` instances cannot be compared reliably when some have 

206 integers IDs and others do not. 

207 """ 

208 return self._outputs 

209 

210 def addPredictedInput(self, ref: DatasetRef) -> None: 

211 """Add an input `DatasetRef` to the `Quantum`. 

212 

213 This does not automatically update a `Registry`; all `predictedInputs` 

214 must be present before a `Registry.addQuantum()` is called. 

215 

216 Parameters 

217 ---------- 

218 ref : `DatasetRef` 

219 Reference for a Dataset to add to the Quantum's predicted inputs. 

220 """ 

221 self._predictedInputs.setdefault(ref.datasetType, []).append(ref) 

222 

223 def _markInputUsed(self, ref: DatasetRef) -> None: 

224 """Mark an input as used. 

225 

226 This does not automatically update a `Registry`. 

227 For that use `Registry.markInputUsed()` instead. 

228 """ 

229 # First validate against predicted 

230 if ref.datasetType not in self._predictedInputs: 

231 raise ValueError(f"Dataset type {ref.datasetType.name} not in predicted inputs") 

232 if ref not in self._predictedInputs[ref.datasetType]: 

233 raise ValueError(f"Actual input {ref} was not predicted") 

234 # Now insert as actual 

235 self._actualInputs.setdefault(ref.datasetType, []).append(ref) 

236 

237 def addOutput(self, ref: DatasetRef) -> None: 

238 """Add an output `DatasetRef` to the `Quantum`. 

239 

240 This does not automatically update a `Registry`; all `outputs` 

241 must be present before a `Registry.addQuantum()` is called. 

242 

243 Parameters 

244 ---------- 

245 ref : `DatasetRef` 

246 Reference for a Dataset to add to the Quantum's outputs. 

247 """ 

248 self._outputs.setdefault(ref.datasetType, []).append(ref) 

249 

250 @property 

251 def id(self) -> Optional[int]: 

252 """Unique (autoincrement) integer for this quantum (`int`). 

253 """ 

254 return self._id 

255 

256 @property 

257 def startTime(self) -> Optional[astropy.time.Time]: 

258 """Begin timestamp for the execution of this quantum 

259 (`astropy.time.Time`). 

260 """ 

261 return self._startTime 

262 

263 @property 

264 def endTime(self) -> Optional[astropy.time.Time]: 

265 """End timestamp for the execution of this quantum 

266 (`astropy.time.Time`). 

267 """ 

268 return self._endTime 

269 

270 @property 

271 def host(self) -> Optional[str]: 

272 """Name of the system on which this quantum was executed (`str`). 

273 """ 

274 return self._host