Coverage for python/lsst/daf/butler/core/quantum.py : 28%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Quantum",)
26from typing import (
27 Iterable,
28 List,
29 Mapping,
30 Optional,
31 Type,
32 TYPE_CHECKING,
33 Union,
34)
36import astropy.time
38from lsst.utils import doImport
40from .named import NamedKeyDict
42if TYPE_CHECKING: 42 ↛ 43line 42 didn't jump to line 43, because the condition on line 42 was never true
43 from .dimensions import DataCoordinate
44 from .datasets import DatasetRef, DatasetType
47class Quantum:
48 """A discrete unit of work that may depend on one or more datasets and
49 produces one or more datasets.
51 Most Quanta will be executions of a particular ``PipelineTask``’s
52 ``runQuantum`` method, but they can also be used to represent discrete
53 units of work performed manually by human operators or other software
54 agents.
56 Parameters
57 ----------
58 taskName : `str`, optional
59 Fully-qualified name of the Task class that executed or will execute
60 this Quantum. If not provided, ``taskClass`` must be.
61 taskClass : `type`, optional
62 The Task class that executed or will execute this Quantum. If not
63 provided, ``taskName`` must be. Overrides ``taskName`` if both are
64 provided.
65 dataId : `DataId`, optional
66 The dimension values that identify this `Quantum`.
67 run : `str`, optional
68 The name of the run this Quantum is a part of.
69 initInputs : collection of `DatasetRef`, optional
70 Datasets that are needed to construct an instance of the Task. May
71 be a flat iterable of `DatasetRef` instances or a mapping from
72 `DatasetType` to `DatasetRef`.
73 predictedInputs : `~collections.abc.Mapping`, optional
74 Inputs identified prior to execution, organized as a mapping from
75 `DatasetType` to a list of `DatasetRef`. Must be a superset of
76 ``actualInputs``.
77 actualInputs : `~collections.abc.Mapping`, optional
78 Inputs actually used during execution, organized as a mapping from
79 `DatasetType` to a list of `DatasetRef`. Must be a subset of
80 ``predictedInputs``.
81 outputs : `~collections.abc.Mapping`, optional
82 Outputs from executing this quantum of work, organized as a mapping
83 from `DatasetType` to a list of `DatasetRef`.
84 startTime : `astropy.time.Time`
85 The start time for the quantum.
86 endTime : `astropy.time.Time`
87 The end time for the quantum.
88 host : `str`
89 The system on this quantum was executed.
90 id : `int`, optional
91 Unique integer identifier for this quantum. Usually set to `None`
92 (default) and assigned by `Registry`.
93 """
95 __slots__ = ("_taskName", "_taskClass", "_dataId", "_run",
96 "_initInputs", "_predictedInputs", "_actualInputs", "_outputs",
97 "_id", "_startTime", "_endTime", "_host")
99 def __init__(self, *, taskName: Optional[str] = None,
100 taskClass: Optional[Type] = None,
101 dataId: Optional[DataCoordinate] = None,
102 run: Optional[str] = None,
103 initInputs: Optional[Union[Mapping[DatasetType, DatasetRef], Iterable[DatasetRef]]] = None,
104 predictedInputs: Optional[Mapping[DatasetType, List[DatasetRef]]] = None,
105 actualInputs: Optional[Mapping[DatasetType, List[DatasetRef]]] = None,
106 outputs: Optional[Mapping[DatasetType, List[DatasetRef]]] = None,
107 startTime: Optional[astropy.time.Time] = None,
108 endTime: Optional[astropy.time.Time] = None,
109 host: Optional[str] = None,
110 id: Optional[int] = None):
111 if taskClass is not None:
112 taskName = f"{taskClass.__module__}.{taskClass.__name__}"
113 self._taskName = taskName
114 self._taskClass = taskClass
115 self._run = run
116 self._dataId = dataId
117 if initInputs is None:
118 initInputs = {}
119 elif not isinstance(initInputs, Mapping):
120 initInputs = {ref.datasetType: ref for ref in initInputs}
121 if predictedInputs is None:
122 predictedInputs = {}
123 if actualInputs is None:
124 actualInputs = {}
125 if outputs is None:
126 outputs = {}
127 self._initInputs: NamedKeyDict[DatasetType, DatasetRef] = NamedKeyDict(initInputs)
128 self._predictedInputs: NamedKeyDict[DatasetType, List[DatasetRef]] = NamedKeyDict(predictedInputs)
129 self._actualInputs: NamedKeyDict[DatasetType, List[DatasetRef]] = NamedKeyDict(actualInputs)
130 self._outputs: NamedKeyDict[DatasetType, List[DatasetRef]] = NamedKeyDict(outputs)
131 self._id = id
132 self._startTime = startTime
133 self._endTime = endTime
134 self._host = host
136 @property
137 def taskClass(self) -> Optional[Type]:
138 """Task class associated with this `Quantum` (`type`).
139 """
140 if self._taskClass is None:
141 self._taskClass = doImport(self._taskName)
142 return self._taskClass
144 @property
145 def taskName(self) -> Optional[str]:
146 """Fully-qualified name of the task associated with `Quantum` (`str`).
147 """
148 return self._taskName
150 @property
151 def run(self) -> Optional[str]:
152 """The name of the run this Quantum is a part of (`str`).
153 """
154 return self._run
156 @property
157 def dataId(self) -> Optional[DataCoordinate]:
158 """The dimension values of the unit of processing (`DataId`).
159 """
160 return self._dataId
162 @property
163 def initInputs(self) -> NamedKeyDict[DatasetType, DatasetRef]:
164 """A mapping of datasets used to construct the Task,
165 with `DatasetType` instances as keys (names can also be used for
166 lookups) and `DatasetRef` instances as values.
167 """
168 return self._initInputs
170 @property
171 def predictedInputs(self) -> NamedKeyDict[DatasetType, List[DatasetRef]]:
172 """A mapping of input datasets that were expected to be used,
173 with `DatasetType` instances as keys (names can also be used for
174 lookups) and a list of `DatasetRef` instances as values.
176 Notes
177 -----
178 We cannot use `set` instead of `list` for the nested container because
179 `DatasetRef` instances cannot be compared reliably when some have
180 integers IDs and others do not.
181 """
182 return self._predictedInputs
184 @property
185 def actualInputs(self) -> NamedKeyDict[DatasetType, List[DatasetRef]]:
186 """A mapping of input datasets that were actually used, with the same
187 form as `Quantum.predictedInputs`.
189 Notes
190 -----
191 We cannot use `set` instead of `list` for the nested container because
192 `DatasetRef` instances cannot be compared reliably when some have
193 integers IDs and others do not.
194 """
195 return self._actualInputs
197 @property
198 def outputs(self) -> NamedKeyDict[DatasetType, List[DatasetRef]]:
199 """A mapping of output datasets (to be) generated for this quantum,
200 with the same form as `predictedInputs`.
202 Notes
203 -----
204 We cannot use `set` instead of `list` for the nested container because
205 `DatasetRef` instances cannot be compared reliably when some have
206 integers IDs and others do not.
207 """
208 return self._outputs
210 def addPredictedInput(self, ref: DatasetRef) -> None:
211 """Add an input `DatasetRef` to the `Quantum`.
213 This does not automatically update a `Registry`; all `predictedInputs`
214 must be present before a `Registry.addQuantum()` is called.
216 Parameters
217 ----------
218 ref : `DatasetRef`
219 Reference for a Dataset to add to the Quantum's predicted inputs.
220 """
221 self._predictedInputs.setdefault(ref.datasetType, []).append(ref)
223 def _markInputUsed(self, ref: DatasetRef) -> None:
224 """Mark an input as used.
226 This does not automatically update a `Registry`.
227 For that use `Registry.markInputUsed()` instead.
228 """
229 # First validate against predicted
230 if ref.datasetType not in self._predictedInputs:
231 raise ValueError(f"Dataset type {ref.datasetType.name} not in predicted inputs")
232 if ref not in self._predictedInputs[ref.datasetType]:
233 raise ValueError(f"Actual input {ref} was not predicted")
234 # Now insert as actual
235 self._actualInputs.setdefault(ref.datasetType, []).append(ref)
237 def addOutput(self, ref: DatasetRef) -> None:
238 """Add an output `DatasetRef` to the `Quantum`.
240 This does not automatically update a `Registry`; all `outputs`
241 must be present before a `Registry.addQuantum()` is called.
243 Parameters
244 ----------
245 ref : `DatasetRef`
246 Reference for a Dataset to add to the Quantum's outputs.
247 """
248 self._outputs.setdefault(ref.datasetType, []).append(ref)
250 @property
251 def id(self) -> Optional[int]:
252 """Unique (autoincrement) integer for this quantum (`int`).
253 """
254 return self._id
256 @property
257 def startTime(self) -> Optional[astropy.time.Time]:
258 """Begin timestamp for the execution of this quantum
259 (`astropy.time.Time`).
260 """
261 return self._startTime
263 @property
264 def endTime(self) -> Optional[astropy.time.Time]:
265 """End timestamp for the execution of this quantum
266 (`astropy.time.Time`).
267 """
268 return self._endTime
270 @property
271 def host(self) -> Optional[str]:
272 """Name of the system on which this quantum was executed (`str`).
273 """
274 return self._host