Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("Quantum",) 

23 

24import astropy.time 

25 

26from lsst.utils import doImport 

27 

28from .utils import NamedKeyDict 

29 

30 

31class Quantum: 

32 """A discrete unit of work that may depend on one or more datasets and 

33 produces one or more datasets. 

34 

35 Most Quanta will be executions of a particular ``PipelineTask``’s 

36 ``runQuantum`` method, but they can also be used to represent discrete 

37 units of work performed manually by human operators or other software 

38 agents. 

39 

40 Parameters 

41 ---------- 

42 taskName : `str`, optional 

43 Fully-qualified name of the Task class that executed or will execute 

44 this Quantum. If not provided, ``taskClass`` must be. 

45 taskClass : `type`, optional 

46 The Task class that executed or will execute this Quantum. If not 

47 provided, ``taskName`` must be. Overrides ``taskName`` if both are 

48 provided. 

49 dataId : `DataId`, optional 

50 The dimension values that identify this `Quantum`. 

51 run : `str`, optional 

52 The name of the run this Quantum is a part of. 

53 initInputs : collection of `DatasetRef`, optional 

54 Datasets that are needed to construct an instance of the Task. May 

55 be a flat iterable of `DatasetRef` instances or a mapping from 

56 `DatasetType` to `DatasetRef`. 

57 predictedInputs : `~collections.abc.Mapping`, optional 

58 Inputs identified prior to execution, organized as a mapping from 

59 `DatasetType` to a list of `DatasetRef`. Must be a superset of 

60 ``actualInputs``. 

61 actualInputs : `~collections.abc.Mapping`, optional 

62 Inputs actually used during execution, organized as a mapping from 

63 `DatasetType` to a list of `DatasetRef`. Must be a subset of 

64 ``predictedInputs``. 

65 outputs : `~collections.abc.Mapping`, optional 

66 Outputs from executing this quantum of work, organized as a mapping 

67 from `DatasetType` to a list of `DatasetRef`. 

68 startTime : `astropy.time.Time` 

69 The start time for the quantum. 

70 endTime : `astropy.time.Time` 

71 The end time for the quantum. 

72 host : `str` 

73 The system on this quantum was executed. 

74 id : `int`, optional 

75 Unique integer identifier for this quantum. Usually set to `None` 

76 (default) and assigned by `Registry`. 

77 """ 

78 

79 __slots__ = ("_taskName", "_taskClass", "_dataId", "_run", 

80 "_initInputs", "_predictedInputs", "_actualInputs", "_outputs", 

81 "_id", "_startTime", "_endTime", "_host") 

82 

83 def __init__(self, *, taskName=None, taskClass=None, dataId=None, run=None, 

84 initInputs=None, predictedInputs=(), actualInputs=(), outputs=(), 

85 startTime=None, endTime=None, host=None, id=None, 

86 **kwargs): 

87 super().__init__(**kwargs) 

88 if taskClass is not None: 

89 taskName = f"{taskClass.__module__}.{taskClass.__name__}" 

90 self._taskName = taskName 

91 self._taskClass = taskClass 

92 self._run = run 

93 self._dataId = dataId 

94 if initInputs is None: 

95 initInputs = {} 

96 elif not hasattr(initInputs, "keys"): 

97 initInputs = {ref.datasetType: ref for ref in initInputs} 

98 self._initInputs = NamedKeyDict(initInputs) 

99 self._predictedInputs = NamedKeyDict(predictedInputs) 

100 self._actualInputs = NamedKeyDict(actualInputs) 

101 self._outputs = NamedKeyDict(outputs) 

102 self._id = id 

103 self._startTime = startTime 

104 self._endTime = endTime 

105 self._host = host 

106 

107 @property 

108 def taskClass(self): 

109 """Task class associated with this `Quantum` (`type`). 

110 """ 

111 if self._taskClass is None: 

112 self._taskClass = doImport(self._taskName) 

113 return self._taskClass 

114 

115 @property 

116 def taskName(self): 

117 """Fully-qualified name of the task associated with `Quantum` (`str`). 

118 """ 

119 return self._taskName 

120 

121 @property 

122 def run(self): 

123 """The name of the run this Quantum is a part of (`str`). 

124 """ 

125 return self._run 

126 

127 @property 

128 def dataId(self): 

129 """The dimension values of the unit of processing (`DataId`). 

130 """ 

131 return self._dataId 

132 

133 @property 

134 def initInputs(self): 

135 """A mapping of datasets used to construct the Task, 

136 with `DatasetType` instances as keys (names can also be used for 

137 lookups) and `DatasetRef` instances as values. 

138 """ 

139 return self._initInputs 

140 

141 @property 

142 def predictedInputs(self): 

143 """A mapping of input datasets that were expected to be used, 

144 with `DatasetType` instances as keys (names can also be used for 

145 lookups) and a list of `DatasetRef` instances as values. 

146 

147 Notes 

148 ----- 

149 We cannot use `set` instead of `list` for the nested container because 

150 `DatasetRef` instances cannot be compared reliably when some have 

151 integers IDs and others do not. 

152 """ 

153 return self._predictedInputs 

154 

155 @property 

156 def actualInputs(self): 

157 """A mapping of input datasets that were actually used, with the same 

158 form as `Quantum.predictedInputs`. 

159 

160 Notes 

161 ----- 

162 We cannot use `set` instead of `list` for the nested container because 

163 `DatasetRef` instances cannot be compared reliably when some have 

164 integers IDs and others do not. 

165 """ 

166 return self._actualInputs 

167 

168 @property 

169 def outputs(self): 

170 """A mapping of output datasets (to be) generated for this quantum, 

171 with the same form as `predictedInputs`. 

172 

173 Notes 

174 ----- 

175 We cannot use `set` instead of `list` for the nested container because 

176 `DatasetRef` instances cannot be compared reliably when some have 

177 integers IDs and others do not. 

178 """ 

179 return self._outputs 

180 

181 def addPredictedInput(self, ref): 

182 """Add an input `DatasetRef` to the `Quantum`. 

183 

184 This does not automatically update a `Registry`; all `predictedInputs` 

185 must be present before a `Registry.addQuantum()` is called. 

186 

187 Parameters 

188 ---------- 

189 ref : `DatasetRef` 

190 Reference for a Dataset to add to the Quantum's predicted inputs. 

191 """ 

192 self._predictedInputs.setdefault(ref.datasetType, []).append(ref) 

193 

194 def _markInputUsed(self, ref): 

195 """Mark an input as used. 

196 

197 This does not automatically update a `Registry`. 

198 For that use `Registry.markInputUsed()` instead. 

199 """ 

200 # First validate against predicted 

201 if ref.datasetType not in self._predictedInputs: 

202 raise ValueError("Dataset type {} not in predicted inputs".format(ref.datasetType.name)) 

203 if ref not in self._predictedInputs[ref.datasetType]: 

204 raise ValueError("Actual input {} was not predicted".format(ref)) 

205 # Now insert as actual 

206 self._actualInputs.setdefault(ref.datasetType, []).append(ref) 

207 

208 def addOutput(self, ref): 

209 """Add an output `DatasetRef` to the `Quantum`. 

210 

211 This does not automatically update a `Registry`; all `outputs` 

212 must be present before a `Registry.addQuantum()` is called. 

213 

214 Parameters 

215 ---------- 

216 ref : `DatasetRef` 

217 Reference for a Dataset to add to the Quantum's outputs. 

218 """ 

219 self._outputs.setdefault(ref.datasetType, []).append(ref) 

220 

221 @property 

222 def id(self) -> int: 

223 """Unique (autoincrement) integer for this quantum (`int`). 

224 """ 

225 return self._id 

226 

227 @property 

228 def startTime(self) -> astropy.time.Time: 

229 """Begin timestamp for the execution of this quantum 

230 (`astropy.time.Time`). 

231 """ 

232 return self._startTime 

233 

234 @property 

235 def endTime(self) -> astropy.time.Time: 

236 """End timestamp for the execution of this quantum 

237 (`astropy.time.Time`). 

238 """ 

239 return self._endTime 

240 

241 @property 

242 def host(self) -> str: 

243 """Name of the system on which this quantum was executed (`str`). 

244 """ 

245 return self._host