33 """Arguments passed into a Butler that are used to instantiate a repository. This includes arguments that 34 can be used to create a new repository (cfgRoot, root, mapper, mapperArgs, policy) and are persisted along 35 with the new repository's configuration file. These arguments can also describe how a new or existing 36 repository are to be used (cfgRoot or root, tags, mode). When indicating an existing repository it is 37 better to not specify unnecessary arguments, as if they conflict with the persisted repository 38 configuration then a RuntimeError will be raised during Butler init. 40 A RepositoryArgs class can be initialized from a dict, if the first argument to the initializer is a dict. 44 cfgRoot : URI or dict, optional 45 If dict, the initalizer is re-called with the expanded dict. 46 If URI, this is the location where the RepositoryCfg should be found (existing repo) or put (new repo) 48 If different than cfgRoot then this is the location where the repository should exist. A RepositoryCfg 49 will be put at cfgRoot and its root will be a path to root. 50 mapper : string or class object, optional 51 The mapper to use with this repository. If string, should refer an importable object. If class object, 52 should be a mapper to be instantiated by the Butler during Butler init. 54 Arguments & values to pass to the mapper when initializing it. 55 tags : list or object, optional 56 One or more unique identifiers to uniquely identify this repository and its parents when performing 58 mode : string, optional 59 should be one of 'r', 'w', or 'rw', for 'read', 'write', or 'read-write'. Can be omitted; input
60 repositories will default to '
r', output repositories will default to 'w'. 'w' on an input repository
61 will raise a RuntimeError during Butler init, although 'rw' works and is equivalent to '
r'. Output 62 repositories may be 'r' or 'rw', '
r' for an output repository will raise a RuntimeError during Butler 65 Policy associated with this repository, overrides all other policy data (which may be loaded from 66 policies
in derived packages).
68 def __init__(self, cfgRoot=None, root=None, mapper=None, mapperArgs=None, tags=None, 69 mode=None, policy=None): 71 # is cfgRoot a dict? try dict init: 72 self.__init__(**cfgRoot) 74 self._root = Storage.absolutePath(os.getcwd(), root.rstrip(os.sep)) if root else root 75 self._cfgRoot = Storage.absolutePath(os.getcwd(), cfgRoot.rstrip(os.sep)) if cfgRoot else cfgRoot 77 self.mapperArgs = mapperArgs 78 self.tags = set(listify(tags)) 80 self.policy = Policy(policy) if policy is not None else None 83 return "%s(root=%r, cfgRoot=%r, mapper=%r, mapperArgs=%r, tags=%s, mode=%r, policy=%s)" % ( 84 self.__class__.__name__, self.root, self._cfgRoot, self._mapper, self.mapperArgs, self.tags, 85 self.mode, self.policy) 92 def mapper(self, mapper): 93 if mapper is not None and self._mapper: 94 raise RuntimeError("Explicity clear mapper (set to None) before changing its value.") 99 return self._cfgRoot if self._cfgRoot is not None else self._root 103 return self._root if self._root is not None else self._cfgRoot 106 def inputRepo(storage, tags=None): 107 return RepositoryArgs(storage, tags) 110 def outputRepo(storage, mapper=None, mapperArgs=None, tags=None, mode=None): 111 return RepositoryArgs(storage, mapper, mapperArgs, tags, mode) 114 """add a tag to the repository cfg
""" 115 if isinstance(tag, str): 119 self.tags.update(tag) 125 """Represents a repository of persisted data
and has methods to access that data.
128 def __init__(self, repoData): 129 """Initialize a Repository with parameters input via RepoData.
134 Object that contains the parameters with which to init the Repository.
136 self._storage = Storage.makeFromURI(repoData.cfg.root) 137 if repoData.cfg.dirty and not repoData.isV1Repository and repoData.cfgOrigin != 'nested': 138 self._storage.putRepositoryCfg(repoData.cfg, repoData.cfgRoot) 139 self._mapperArgs = repoData.cfg.mapperArgs # keep for reference in matchesArgs 140 self._initMapper(repoData) 142 def _initMapper(self, repoData): 143 '''Initialize and keep the mapper in a member var. 148 The RepoData with the properties of this Repository. 151 # rule: If mapper is: 152 # - an object: use it as the mapper. 153 # - a string: import it and instantiate it with mapperArgs 154 # - a class object: instantiate it with mapperArgs 155 mapper = repoData.cfg.mapper 157 # if mapper is a string, import it: 158 if isinstance(mapper, str): 159 mapper = doImport(mapper) 160 # now if mapper is a class type (not instance), instantiate it: 161 if inspect.isclass(mapper): 162 mapperArgs = copy.copy(repoData.cfg.mapperArgs) 163 if mapperArgs is None: 165 if 'root' not in mapperArgs: 166 mapperArgs['root'] = repoData.cfg.root 167 mapper = mapper(parentRegistry=repoData.parentRegistry, 168 repositoryCfg=repoData.cfg, 170 self._mapper = mapper 172 # todo want a way to make a repository read-only 173 def write(self, butlerLocation, obj): 174 """Write a dataset to Storage.
176 :param butlerLocation: Contains the details needed to find the desired dataset.
177 :param dataset: The dataset to be written.
180 butlerLocationStorage = butlerLocation.getStorage() 181 if butlerLocationStorage: 182 return butlerLocationStorage.write(butlerLocation, obj) 184 return self._storage.write(butlerLocation, obj) 186 def read(self, butlerLocation): 187 """Read a dataset
from Storage.
189 :param butlerLocation: Contains the details needed to find the desired dataset.
190 :
return: An instance of the dataset requested by butlerLocation.
192 butlerLocationStorage = butlerLocation.getStorage() 193 if butlerLocationStorage: 194 return butlerLocationStorage.read(butlerLocation) 196 return self._storage.read(butlerLocation) 202 return (self._mapper, ) 204 def getRegistry(self): 205 """Get the registry
from the mapper
210 The registry
from the mapper
or None if the mapper does
not have one.
212 if self._mapper is None: 214 return self._mapper.getRegistry() 216 def getKeys(self, *args, **kwargs): 218 Get the keys available
in the repository/repositories.
221 :
return: A dict of {key:valueType}
223 # todo: getKeys is not in the mapper API 224 if self._mapper is None: 226 keys = self._mapper.getKeys(*args, **kwargs) 229 def map(self, *args, **kwargs): 230 """Find a butler location
for the given arguments.
231 See mapper.map
for more information about args
and kwargs.
233 :param args: arguments to be passed on to mapper.map
234 :param kwargs: keyword arguments to be passed on to mapper.map
235 :
return: The type of item
is dependent on the mapper being used but
is typically a ButlerLocation.
237 if self._mapper is None: 238 raise RuntimeError("No mapper assigned to Repository") 239 loc = self._mapper.map(*args, **kwargs) 242 loc.setRepository(self) 245 def queryMetadata(self, *args, **kwargs): 246 """Gets possible values
for keys given a partial data id.
248 See mapper documentation
for more explanation about queryMetadata.
250 :param args: arguments to be passed on to mapper.queryMetadata
251 :param kwargs: keyword arguments to be passed on to mapper.queryMetadata
252 :
return:The type of item
is dependent on the mapper being used but
is typically a set that contains
253 available values
for the keys
in the format input argument.
255 if self._mapper is None: 257 ret = self._mapper.queryMetadata(*args, **kwargs) 260 def backup(self, *args, **kwargs): 261 """Perform mapper.backup.
263 See mapper.backup
for more information about args
and kwargs.
265 :param args: arguments to be passed on to mapper.backup
266 :param kwargs: keyword arguments to be passed on to mapper.backup
269 if self._mapper is None: 271 self._mapper.backup(*args, **kwargs) 273 def getMapperDefaultLevel(self): 274 """Get the default level of the mapper.
276 This
is typically used
if no level
is passed into butler methods that call repository.getKeys
and/
or 277 repository.queryMetadata. There
is a bug
in that code because it gets the default level
from this
278 repository but then uses that value when searching all repositories. If this
and other repositories
279 have dissimilar data, the default level value will be nonsensical. A good example of this issue
is in 280 Butler.subset; it needs refactoring.
284 if self._mapper is None: 286 return self._mapper.getDefaultLevel() 288 def exists(self, location): 289 """Check
if location exists
in storage.
293 location : ButlerLocation
294 Desrcibes a location
in storage to look
for.
299 True if location exists,
False if not.
301 butlerLocationStorage = location.getStorage() 302 if butlerLocationStorage: 303 return butlerLocationStorage.exists(location) 305 return self._storage.exists(location)