Coverage for python/lsst/daf/butler/core/quantum.py : 32%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("Quantum",)
24import astropy.time
26from lsst.utils import doImport
28from .utils import NamedKeyDict
31class Quantum:
32 """A discrete unit of work that may depend on one or more datasets and
33 produces one or more datasets.
35 Most Quanta will be executions of a particular ``PipelineTask``’s
36 ``runQuantum`` method, but they can also be used to represent discrete
37 units of work performed manually by human operators or other software
38 agents.
40 Parameters
41 ----------
42 taskName : `str`, optional
43 Fully-qualified name of the Task class that executed or will execute
44 this Quantum. If not provided, ``taskClass`` must be.
45 taskClass : `type`, optional
46 The Task class that executed or will execute this Quantum. If not
47 provided, ``taskName`` must be. Overrides ``taskName`` if both are
48 provided.
49 dataId : `DataId`, optional
50 The dimension values that identify this `Quantum`.
51 run : `str`, optional
52 The name of the run this Quantum is a part of.
53 initInputs : collection of `DatasetRef`, optional
54 Datasets that are needed to construct an instance of the Task. May
55 be a flat iterable of `DatasetRef` instances or a mapping from
56 `DatasetType` to `DatasetRef`.
57 predictedInputs : `~collections.abc.Mapping`, optional
58 Inputs identified prior to execution, organized as a mapping from
59 `DatasetType` to a list of `DatasetRef`. Must be a superset of
60 ``actualInputs``.
61 actualInputs : `~collections.abc.Mapping`, optional
62 Inputs actually used during execution, organized as a mapping from
63 `DatasetType` to a list of `DatasetRef`. Must be a subset of
64 ``predictedInputs``.
65 outputs : `~collections.abc.Mapping`, optional
66 Outputs from executing this quantum of work, organized as a mapping
67 from `DatasetType` to a list of `DatasetRef`.
68 startTime : `astropy.time.Time`
69 The start time for the quantum.
70 endTime : `astropy.time.Time`
71 The end time for the quantum.
72 host : `str`
73 The system on this quantum was executed.
74 id : `int`, optional
75 Unique integer identifier for this quantum. Usually set to `None`
76 (default) and assigned by `Registry`.
77 """
79 __slots__ = ("_taskName", "_taskClass", "_dataId", "_run",
80 "_initInputs", "_predictedInputs", "_actualInputs", "_outputs",
81 "_id", "_startTime", "_endTime", "_host")
83 def __init__(self, *, taskName=None, taskClass=None, dataId=None, run=None,
84 initInputs=None, predictedInputs=(), actualInputs=(), outputs=(),
85 startTime=None, endTime=None, host=None, id=None,
86 **kwargs):
87 super().__init__(**kwargs)
88 if taskClass is not None:
89 taskName = f"{taskClass.__module__}.{taskClass.__name__}"
90 self._taskName = taskName
91 self._taskClass = taskClass
92 self._run = run
93 self._dataId = dataId
94 if initInputs is None:
95 initInputs = {}
96 elif not hasattr(initInputs, "keys"):
97 initInputs = {ref.datasetType: ref for ref in initInputs}
98 self._initInputs = NamedKeyDict(initInputs)
99 self._predictedInputs = NamedKeyDict(predictedInputs)
100 self._actualInputs = NamedKeyDict(actualInputs)
101 self._outputs = NamedKeyDict(outputs)
102 self._id = id
103 self._startTime = startTime
104 self._endTime = endTime
105 self._host = host
107 @property
108 def taskClass(self):
109 """Task class associated with this `Quantum` (`type`).
110 """
111 if self._taskClass is None:
112 self._taskClass = doImport(self._taskName)
113 return self._taskClass
115 @property
116 def taskName(self):
117 """Fully-qualified name of the task associated with `Quantum` (`str`).
118 """
119 return self._taskName
121 @property
122 def run(self):
123 """The name of the run this Quantum is a part of (`str`).
124 """
125 return self._run
127 @property
128 def dataId(self):
129 """The dimension values of the unit of processing (`DataId`).
130 """
131 return self._dataId
133 @property
134 def initInputs(self):
135 """A mapping of datasets used to construct the Task,
136 with `DatasetType` instances as keys (names can also be used for
137 lookups) and `DatasetRef` instances as values.
138 """
139 return self._initInputs
141 @property
142 def predictedInputs(self):
143 """A mapping of input datasets that were expected to be used,
144 with `DatasetType` instances as keys (names can also be used for
145 lookups) and a list of `DatasetRef` instances as values.
147 Notes
148 -----
149 We cannot use `set` instead of `list` for the nested container because
150 `DatasetRef` instances cannot be compared reliably when some have
151 integers IDs and others do not.
152 """
153 return self._predictedInputs
155 @property
156 def actualInputs(self):
157 """A mapping of input datasets that were actually used, with the same
158 form as `Quantum.predictedInputs`.
160 Notes
161 -----
162 We cannot use `set` instead of `list` for the nested container because
163 `DatasetRef` instances cannot be compared reliably when some have
164 integers IDs and others do not.
165 """
166 return self._actualInputs
168 @property
169 def outputs(self):
170 """A mapping of output datasets (to be) generated for this quantum,
171 with the same form as `predictedInputs`.
173 Notes
174 -----
175 We cannot use `set` instead of `list` for the nested container because
176 `DatasetRef` instances cannot be compared reliably when some have
177 integers IDs and others do not.
178 """
179 return self._outputs
181 def addPredictedInput(self, ref):
182 """Add an input `DatasetRef` to the `Quantum`.
184 This does not automatically update a `Registry`; all `predictedInputs`
185 must be present before a `Registry.addQuantum()` is called.
187 Parameters
188 ----------
189 ref : `DatasetRef`
190 Reference for a Dataset to add to the Quantum's predicted inputs.
191 """
192 self._predictedInputs.setdefault(ref.datasetType, []).append(ref)
194 def _markInputUsed(self, ref):
195 """Mark an input as used.
197 This does not automatically update a `Registry`.
198 For that use `Registry.markInputUsed()` instead.
199 """
200 # First validate against predicted
201 if ref.datasetType not in self._predictedInputs:
202 raise ValueError("Dataset type {} not in predicted inputs".format(ref.datasetType.name))
203 if ref not in self._predictedInputs[ref.datasetType]:
204 raise ValueError("Actual input {} was not predicted".format(ref))
205 # Now insert as actual
206 self._actualInputs.setdefault(ref.datasetType, []).append(ref)
208 def addOutput(self, ref):
209 """Add an output `DatasetRef` to the `Quantum`.
211 This does not automatically update a `Registry`; all `outputs`
212 must be present before a `Registry.addQuantum()` is called.
214 Parameters
215 ----------
216 ref : `DatasetRef`
217 Reference for a Dataset to add to the Quantum's outputs.
218 """
219 self._outputs.setdefault(ref.datasetType, []).append(ref)
221 @property
222 def id(self) -> int:
223 """Unique (autoincrement) integer for this quantum (`int`).
224 """
225 return self._id
227 @property
228 def startTime(self) -> astropy.time.Time:
229 """Begin timestamp for the execution of this quantum
230 (`astropy.time.Time`).
231 """
232 return self._startTime
234 @property
235 def endTime(self) -> astropy.time.Time:
236 """End timestamp for the execution of this quantum
237 (`astropy.time.Time`).
238 """
239 return self._endTime
241 @property
242 def host(self) -> str:
243 """Name of the system on which this quantum was executed (`str`).
244 """
245 return self._host