22 """Module defining a butler like object specialized to a specific quantum. 25 __all__ = (
"ButlerQuantumContext",)
30 from .connections
import InputQuantizedConnection, OutputQuantizedConnection, DeferredDatasetRef
31 from .struct
import Struct
32 from lsst.daf.butler
import DatasetRef, Butler, Quantum
36 """Butler like class specialized for a single quantum 38 A ButlerQuantumContext class wraps a standard butler interface and 39 specializes it to the context of a given quantum. What this means 40 in practice is that the only gets and puts that this class allows 41 are DatasetRefs that are contained in the quantum. 43 In the future this class will also be used to record provenance on 44 what was actually get and put. This is in contrast to what the 45 preflight expects to be get and put by looking at the graph before 50 butler : `lsst.daf.butler.Butler` 51 Butler object from/to which datasets will be get/put 52 quantum : `lsst.daf.butler.core.Quantum` 53 Quantum object that describes the datasets which will 54 be get/put by a single execution of this node in the 57 def __init__(self, butler: Butler, quantum: Quantum):
62 for refs
in quantum.predictedInputs.values():
64 self.
allInputs.add((ref.datasetType, ref.dataId))
65 for refs
in quantum.outputs.values():
67 self.
allOutputs.add((ref.datasetType, ref.dataId))
72 if isinstance(ref, DeferredDatasetRef):
74 return butler.getDeferred(ref.datasetRef)
78 return butler.get(ref)
80 def _put(self, value, ref):
82 butler.put(value, ref)
84 self.
_get = types.MethodType(_get, self)
85 self.
_put = types.MethodType(_put, self)
87 def get(self, dataset: typing.Union[InputQuantizedConnection,
88 typing.List[DatasetRef],
89 DatasetRef]) -> object:
90 """Fetches data from the butler 95 This argument may either be an `InputQuantizedConnection` which describes 96 all the inputs of a quantum, a list of `~lsst.daf.butler.DatasetRef`, or 97 a single `~lsst.daf.butler.DatasetRef`. The function will get and return 98 the corresponding datasets from the butler. 103 This function returns arbitrary objects fetched from the bulter. The 104 structure these objects are returned in depends on the type of the input 105 argument. If the input dataset argument is a InputQuantizedConnection, then 106 the return type will be a dictionary with keys corresponding to the attributes 107 of the `InputQuantizedConnection` (which in turn are the attribute identifiers 108 of the connections). If the input argument is of type `list` of 109 `~lsst.daf.butler.DatasetRef` then the return type will be a list of objects. 110 If the input argument is a single `~lsst.daf.butler.DatasetRef` then a single 111 object will be returned. 116 If a `DatasetRef` is passed to get that is not defined in the quantum object 118 if isinstance(dataset, InputQuantizedConnection):
120 for name, ref
in dataset:
121 if isinstance(ref, list):
122 val = [self.
_get(r)
for r
in ref]
127 elif isinstance(dataset, list):
128 return [self.
_get(x)
for x
in dataset]
129 elif isinstance(dataset, DatasetRef)
or isinstance(dataset, DeferredDatasetRef):
130 return self.
_get(dataset)
132 raise TypeError(
"Dataset argument is not a type that can be used to get")
134 def put(self, values: typing.Union[Struct, typing.List[typing.Any], object],
135 dataset: typing.Union[OutputQuantizedConnection, typing.List[DatasetRef], DatasetRef]):
136 """Puts data into the butler 140 values : `Struct` or `list` of `object` or `object` 141 The data that should be put with the butler. If the type of the dataset 142 is `OutputQuantizedConnection` then this argument should be a `Struct` 143 with corresponding attribute names. Each attribute should then correspond 144 to either a list of object or a single object depending of the type of the 145 corresponding attribute on dataset. I.e. if dataset.calexp is [datasetRef1, 146 datasetRef2] then values.calexp should be [calexp1, calexp2]. Like wise 147 if there is a single ref, then only a single object need be passed. The same 148 restriction applies if dataset is directly a `list` of `DatasetRef` or a 151 This argument may either be an `InputQuantizedConnection` which describes 152 all the inputs of a quantum, a list of `lsst.daf.butler.DatasetRef`, or 153 a single `lsst.daf.butler.DatasetRef`. The function will get and return 154 the corresponding datasets from the butler. 159 If a `DatasetRef` is passed to put that is not defined in the quantum object, or 160 the type of values does not match what is expected from the type of dataset. 162 if isinstance(dataset, OutputQuantizedConnection):
163 if not isinstance(values, Struct):
164 raise ValueError(
"dataset is a OutputQuantizedConnection, a Struct with corresponding" 165 " attributes must be passed as the values to put")
166 for name, refs
in dataset:
167 valuesAttribute = getattr(values, name)
168 if isinstance(refs, list):
169 if len(refs) != len(valuesAttribute):
170 raise ValueError(f
"There must be a object to put for every Dataset ref in {name}")
171 for i, ref
in enumerate(refs):
172 self.
_put(valuesAttribute[i], ref)
174 self.
_put(valuesAttribute, refs)
175 elif isinstance(dataset, list):
176 if len(dataset) != len(values):
177 raise ValueError(
"There must be a common number of references and values to put")
178 for i, ref
in enumerate(dataset):
179 self.
_put(values[i], ref)
180 elif isinstance(dataset, DatasetRef):
181 self.
_put(values, dataset)
183 raise TypeError(
"Dataset argument is not a type that can be used to put")
185 def _checkMembership(self, ref: typing.Union[typing.List[DatasetRef], DatasetRef], inout: set):
186 """Internal function used to check if a DatasetRef is part of the input quantum 188 This function will raise an exception if the ButlerQuantumContext is used to 189 get/put a DatasetRef which is not defined in the quantum. 193 ref : `list` of `DatasetRef` or `DatasetRef` 194 Either a list or a single `DatasetRef` to check 196 The connection type to check, e.g. either an input or an output. This prevents 197 both types needing to be checked for every operation, which may be important 198 for Quanta with lots of `DatasetRef`s. 200 if not isinstance(ref, list):
203 if (r.datasetType, r.dataId)
not in inout:
204 raise ValueError(
"DatasetRef is not part of the Quantum being processed")