27 """This module defines the ButlerSubset class and the ButlerDataRefs contained 28 within it as well as an iterator over the subset.""" 29 from builtins
import next
30 from builtins
import range
31 from builtins
import object
38 """ButlerSubset is a container for ButlerDataRefs. It represents a 39 collection of data ids that can be used to obtain datasets of the type 40 used when creating the collection or a compatible dataset type. It can be 41 thought of as the result of a query for datasets matching a partial data 44 The ButlerDataRefs are generated at a specified level of the data id 45 hierarchy. If that is not the level at which datasets are specified, the 46 ButlerDataRef.subItems() method may be used to dive further into the 49 ButlerSubsets should generally be created using Butler.subset(). 51 This mechanism replaces the creation of butlers using partial dataIds. 55 __init__(self, butler, datasetType, level, dataId) 63 def __init__(self, butler, datasetType, level, dataId):
65 Create a ButlerSubset by querying a butler for data ids matching a 66 given partial data id for a given dataset type at a given hierarchy 69 @param butler (Butler) butler that is being queried. 70 @param datasetType (str) the type of dataset to query. 71 @param level (str) the hierarchy level to descend to. if empty string will look up the default 73 @param dataId (dict) the (partial or complete) data id. 81 keys = self.
butler.getKeys(datasetType, level, tag=dataId.tag)
84 fmt = list(keys.keys())
93 self.
cache.append(dataId)
97 for idTuple
in idTuples:
100 tempId[fmt[0]] = idTuple
102 for i
in range(len(fmt)):
103 tempId[fmt[i]] = idTuple[i]
104 self.
cache.append(tempId)
107 return "ButlerSubset(butler=%s, datasetType=%s, dataId=%s, cache=%s, level=%s)" % (
112 Number of ButlerDataRefs in the ButlerSubset. 117 return len(self.
cache)
121 Iterator over the ButlerDataRefs in the ButlerSubset. 123 @returns (ButlerIterator) 131 An iterator over the ButlerDataRefs in a ButlerSubset. 147 A ButlerDataRef is a reference to a potential dataset or group of datasets 148 that is portable between compatible dataset types. As such, it can be 149 used to create or retrieve datasets. 151 ButlerDataRefs are (conceptually) created as elements of a ButlerSubset by 152 Butler.subset(). They are initially specific to the dataset type passed 153 to that call, but they may be used with any other compatible dataset type. 154 Dataset type compatibility must be determined externally (or by trial and 157 ButlerDataRefs may be created at any level of a data identifier hierarchy. 158 If the level is not one at which datasets exist, a ButlerSubset 159 with lower-level ButlerDataRefs can be created using 160 ButlerDataRef.subItems(). 164 get(self, datasetType=None, **rest) 166 put(self, obj, datasetType=None, **rest) 168 subItems(self, level=None) 170 datasetExists(self, datasetType=None, **rest) 177 For internal use only. ButlerDataRefs should only be created by 178 ButlerSubset and ButlerSubsetIterator. 187 def get(self, datasetType=None, **rest):
189 Retrieve a dataset of the given type (or the type used when creating 190 the ButlerSubset, if None) as specified by the ButlerDataRef. 192 @param datasetType (str) dataset type to retrieve. 193 @param **rest keyword arguments with data identifiers 194 @returns object corresponding to the given dataset type. 196 if datasetType
is None:
200 def put(self, obj, datasetType=None, doBackup=False, **rest):
202 Persist a dataset of the given type (or the type used when creating 203 the ButlerSubset, if None) as specified by the ButlerDataRef. 205 @param obj object to persist. 206 @param datasetType (str) dataset type to persist. 207 @param doBackup if True, rename existing instead of overwriting 208 @param **rest keyword arguments with data identifiers 210 WARNING: Setting doBackup=True is not safe for parallel processing, as it 211 may be subject to race conditions. 214 if datasetType
is None:
216 self.
butlerSubset.butler.put(obj, datasetType, self.
dataId, doBackup=doBackup, **rest)
220 Return a list of the lower levels of the hierarchy than this 223 @returns (iterable) list of strings with level keys.""" 238 Generate a ButlerSubset at a lower level of the hierarchy than this 239 ButlerDataRef, using it as a partial data id. If level is None, a 240 default lower level for the original ButlerSubset level and dataset 243 As currently implemented, the default sublevels for all the 244 repositories used by this Butler instance must match for the Butler to 245 be able to select a default sublevel to get the subset. 247 @param level (str) the hierarchy level to descend to. 248 @returns (ButlerSubset) resulting from the lower-level query or () if 249 there is no lower level. 255 levelSet.add(repoData.repo._mapper.getDefaultSubLevel(
257 if len(levelSet) > 1:
259 "Support for multiple levels not implemented.")
260 level = levelSet.pop()
268 Determine if a dataset exists of the given type (or the type used when 269 creating the ButlerSubset, if None) as specified by the ButlerDataRef. 271 @param datasetType (str) dataset type to check. 272 @param write (bool) if True, search only in output repositories 273 @param **rest keywords arguments with data identifiers 276 if datasetType
is None:
279 datasetType, self.
dataId, write=write, **rest)
283 Return the butler associated with this data reference.
def __init__(self, butlerSubset, dataId)
def __init__(self, butlerSubset)
def __init__(self, butler, datasetType, level, dataId)
def put(self, obj, datasetType=None, doBackup=False, rest)
def datasetExists(self, datasetType=None, write=False, rest)
def subItems(self, level=None)
def get(self, datasetType=None, rest)