22 """Module defining a butler like object specialized to a specific quantum. 25 __all__ = (
"ButlerQuantumContext",)
30 from .connections
import InputQuantizedConnection, OutputQuantizedConnection, DeferredDatasetRef
31 from .struct
import Struct
32 from lsst.daf.butler
import DatasetRef, Butler, Quantum
36 """Butler like class specialized for a single quantum 38 A ButlerQuantumContext class wraps a standard butler interface and 39 specializes it to the context of a given quantum. What this means 40 in practice is that the only gets and puts that this class allows 41 are DatasetRefs that are contained in the quantum. 43 In the future this class will also be used to record provenance on 44 what was actually get and put. This is in contrast to what the 45 preflight expects to be get and put by looking at the graph before 50 butler : `lsst.daf.butler.Butler` 51 Butler object from/to which datasets will be get/put 52 quantum : `lsst.daf.butler.core.Quantum` 53 Quantum object that describes the datasets which will 54 be get/put by a single execution of this node in the 57 def __init__(self, butler: Butler, quantum: Quantum):
62 for refs
in quantum.predictedInputs.values():
64 self.
allInputs.add((ref.datasetType, ref.dataId))
65 for refs
in quantum.outputs.values():
67 self.
allOutputs.add((ref.datasetType, ref.dataId))
72 if isinstance(ref, DeferredDatasetRef):
74 return butler.getDeferred(ref.datasetRef)
78 return butler.get(ref)
80 def _put(self, value, ref):
82 butler.put(value, ref)
84 self.
_get = types.MethodType(_get, self)
85 self.
_put = types.MethodType(_put, self)
87 def get(self, dataset: typing.Union[InputQuantizedConnection,
88 typing.List[DatasetRef],
89 DatasetRef]) -> object:
90 """Fetches data from the butler 94 dataset : `InputQuantizedConnection` or `list` [`~lsst.daf.butler.DatasetRef`] 95 or `~lsst.daf.butler.DatasetRef` 96 This argument may either be an `InputQuantizedConnection` which describes 97 all the inputs of a quantum, a list of `~lsst.daf.butler.DatasetRef`, or 98 a single `~lsst.daf.butler.DatasetRef`. The function will get and return 99 the corresponding datasets from the butler. 104 This function returns arbitrary objects fetched from the bulter. The 105 structure these objects are returned in depends on the type of the input 106 argument. If the input dataset argument is a InputQuantizedConnection, then 107 the return type will be a dictionary with keys corresponding to the attributes 108 of the `InputQuantizedConnection` (which in turn are the attribute identifiers 109 of the connections). If the input argument is of type `list` of 110 `~lsst.daf.butler.DatasetRef` then the return type will be a list of objects. 111 If the input argument is a single `~lsst.daf.butler.DatasetRef` then a single 112 object will be returned. 117 If a `DatasetRef` is passed to get that is not defined in the quantum object 119 if isinstance(dataset, InputQuantizedConnection):
121 for name, ref
in dataset:
122 if isinstance(ref, list):
123 val = [self.
_get(r)
for r
in ref]
128 elif isinstance(dataset, list):
129 return [self.
_get(x)
for x
in dataset]
130 elif isinstance(dataset, DatasetRef)
or isinstance(dataset, DeferredDatasetRef):
131 return self.
_get(dataset)
133 raise TypeError(
"Dataset argument is not a type that can be used to get")
135 def put(self, values: typing.Union[Struct, typing.List[typing.Any], object],
136 dataset: typing.Union[OutputQuantizedConnection, typing.List[DatasetRef], DatasetRef]):
137 """Puts data into the butler 141 values : `Struct` or `list` of `object` or `object` 142 The data that should be put with the butler. If the type of the dataset 143 is `OutputQuantizedConnection` then this argument should be a `Struct` 144 with corresponding attribute names. Each attribute should then correspond 145 to either a list of object or a single object depending of the type of the 146 corresponding attribute on dataset. I.e. if dataset.calexp is [datasetRef1, 147 datasetRef2] then values.calexp should be [calexp1, calexp2]. Like wise 148 if there is a single ref, then only a single object need be passed. The same 149 restriction applies if dataset is directly a `list` of `DatasetRef` or a 151 dataset : `OutputQuantizedConnection` or `list` of `lsst.daf.butler.DatasetRef` 152 or `lsst.daf.butler.DatasetRef` 153 This argument may either be an `InputQuantizedConnection` which describes 154 all the inputs of a quantum, a list of `lsst.daf.butler.DatasetRef`, or 155 a single `lsst.daf.butler.DatasetRef`. The function will get and return 156 the corresponding datasets from the butler. 161 If a `DatasetRef` is passed to put that is not defined in the quantum object, or 162 the type of values does not match what is expected from the type of dataset. 164 if isinstance(dataset, OutputQuantizedConnection):
165 if not isinstance(values, Struct):
166 raise ValueError(
"dataset is a OutputQuantizedConnection, a Struct with corresponding" 167 " attributes must be passed as the values to put")
168 for name, refs
in dataset:
169 valuesAttribute = getattr(values, name)
170 if isinstance(refs, list):
171 if len(refs) != len(valuesAttribute):
172 raise ValueError(f
"There must be a object to put for every Dataset ref in {name}")
173 for i, ref
in enumerate(refs):
174 self.
_put(valuesAttribute[i], ref)
176 self.
_put(valuesAttribute, refs)
177 elif isinstance(dataset, list):
178 if len(dataset) != len(values):
179 raise ValueError(
"There must be a common number of references and values to put")
180 for i, ref
in enumerate(dataset):
181 self.
_put(values[i], ref)
182 elif isinstance(dataset, DatasetRef):
183 self.
_put(values, dataset)
185 raise TypeError(
"Dataset argument is not a type that can be used to put")
187 def _checkMembership(self, ref: typing.Union[typing.List[DatasetRef], DatasetRef], inout: set):
188 """Internal function used to check if a DatasetRef is part of the input quantum 190 This function will raise an exception if the ButlerQuantumContext is used to 191 get/put a DatasetRef which is not defined in the quantum. 195 ref : `list` of `DatasetRef` or `DatasetRef` 196 Either a list or a single `DatasetRef` to check 198 The connection type to check, e.g. either an input or an output. This prevents 199 both types needing to be checked for every operation, which may be important 200 for Quanta with lots of `DatasetRef`s. 202 if not isinstance(ref, list):
205 if (r.datasetType, r.dataId)
not in inout:
206 raise ValueError(
"DatasetRef is not part of the Quantum being processed")