24 from past.builtins
import basestring
25 from builtins
import object
36 """Arguments passed into a Butler that are used to instantiate a repository. This includes arguments that
37 can be used to create a new repository (cfgRoot, root, mapper, mapperArgs, policy) and are persisted along
38 with the new repository's configuration file. These arguments can also describe how a new or existing
39 repository are to be used (cfgRoot or root, tags, mode). When indicating an existing repository it is
40 better to not specify unnecessary arguments, as if they conflict with the persisted repository
41 configuration then a RuntimeError will be raised during Butler init.
43 A RepositoryArgs class can be initialized from a dict, if the first argument to the initializer is a dict.
47 cfgRoot : URI or dict, optional
48 If dict, the initalizer is re-called with the expanded dict.
49 If URI, this is the location where the RepositoryCfg should be found (existing repo) or put (new repo)
51 If different than cfgRoot then this is the location where the repository should exist. A RepositoryCfg
52 will be put at cfgRoot and its root will be a path to root.
53 mapper : string or class object, optional
54 The mapper to use with this repository. If string, should refer an importable object. If class object,
55 should be a mapper to be instantiated by the Butler during Butler init.
57 Arguments & values to pass to the mapper when initializing it.
58 tags : list or object, optional
59 One or more unique identifiers to uniquely identify this repository and its parents when performing
61 mode : string, optional
62 should be one of 'r', 'w', or 'rw', for 'read', 'write', or 'read-write'. Can be omitted; input
63 repositories will default to '
r', output repositories will default to 'w'. 'w' on an input repository
64 will raise a RuntimeError during Butler init, although 'rw' works and is equivalent to '
r'. Output
65 repositories may be 'r' or 'rw', '
r' for an output repository will raise a RuntimeError during Butler
68 Policy associated with this repository, overrides all other policy data (which may be loaded from
69 policies
in derived packages).
71 def __init__(self, cfgRoot=None, root=None, mapper=None, mapperArgs=None, tags=None,
72 mode=None, policy=None):
74 # is cfgRoot a dict? try dict init:
75 self.__init__(**cfgRoot)
77 self._root = Storage.absolutePath(os.getcwd(), root.rstrip(os.sep)) if root else root
78 self._cfgRoot = Storage.absolutePath(os.getcwd(), cfgRoot.rstrip(os.sep)) if cfgRoot else cfgRoot
80 self.mapperArgs = mapperArgs
81 self.tags = set(listify(tags))
83 self.policy = Policy(policy) if policy is not None else None
86 return "%s(root=%r, cfgRoot=%r, mapper=%r, mapperArgs=%r, tags=%s, mode=%r, policy=%s)" % (
87 self.__class__.__name__, self.root, self._cfgRoot, self._mapper, self.mapperArgs, self.tags,
88 self.mode, self.policy)
95 def mapper(self, mapper):
96 if mapper is not None and self._mapper:
97 raise RuntimeError("Explicity clear mapper (set to None) before changing its value.")
102 return self._cfgRoot if self._cfgRoot is not None else self._root
106 return self._root if self._root is not None else self._cfgRoot
109 def inputRepo(storage, tags=None):
110 return RepositoryArgs(storage, tags)
113 def outputRepo(storage, mapper=None, mapperArgs=None, tags=None, mode=None):
114 return RepositoryArgs(storage, mapper, mapperArgs, tags, mode)
117 """add a tag to the repository cfg
"""
118 if isinstance(tag, basestring):
122 self.tags.update(tag)
127 class Repository(object):
128 """Represents a repository of persisted data
and has methods to access that data.
131 def __init__(self, repoData):
132 """Initialize a Repository with parameters input via RepoData.
137 Object that contains the parameters with which to init the Repository.
139 self._storage = Storage.makeFromURI(repoData.cfg.root)
140 if repoData.cfg.dirty and not repoData.isV1Repository and repoData.cfgOrigin != 'nested':
141 self._storage.putRepositoryCfg(repoData.cfg, repoData.cfgRoot)
142 self._mapperArgs = repoData.cfg.mapperArgs # keep for reference in matchesArgs
143 self._initMapper(repoData)
145 def _initMapper(self, repoData):
146 '''Initialize and keep the mapper in a member var.
151 The RepoData with the properties of this Repository.
154 # rule: If mapper is:
155 # - an object: use it as the mapper.
156 # - a string: import it and instantiate it with mapperArgs
157 # - a class object: instantiate it with mapperArgs
158 mapper = repoData.cfg.mapper
160 # if mapper is a string, import it:
161 if isinstance(mapper, basestring):
162 mapper = doImport(mapper)
163 # now if mapper is a class type (not instance), instantiate it:
164 if inspect.isclass(mapper):
165 mapperArgs = copy.copy(repoData.cfg.mapperArgs)
166 if mapperArgs is None:
168 if 'root' not in mapperArgs:
169 mapperArgs['root'] = repoData.cfg.root
170 mapper = mapper(parentRegistry=repoData.parentRegistry,
171 repositoryCfg=repoData.cfg,
173 self._mapper = mapper
176 return 'config(id=%s, storage=%s, parent=%s, mapper=%s, mapperArgs=%s, cls=%s)' % \
177 (self.id, self._storage, self.parent, self._mapper, self.mapperArgs, self.cls)
179 # todo want a way to make a repository read-only
180 def write(self, butlerLocation, obj):
181 """Write a dataset to Storage.
183 :param butlerLocation: Contains the details needed to find the desired dataset.
184 :param dataset: The dataset to be written.
187 butlerLocationStorage = butlerLocation.getStorage()
188 if butlerLocationStorage:
189 return butlerLocationStorage.write(butlerLocation, obj)
191 return self._storage.write(butlerLocation, obj)
193 def read(self, butlerLocation):
194 """Read a dataset
from Storage.
196 :param butlerLocation: Contains the details needed to find the desired dataset.
197 :
return: An instance of the dataset requested by butlerLocation.
199 butlerLocationStorage = butlerLocation.getStorage()
200 if butlerLocationStorage:
201 return butlerLocationStorage.read(butlerLocation)
203 return self._storage.read(butlerLocation)
209 return (self._mapper, )
211 def getRegistry(self):
212 """Get the registry
from the mapper
217 The registry
from the mapper
or None if the mapper does
not have one.
219 if self._mapper is None:
221 return self._mapper.getRegistry()
223 def getKeys(self, *args, **kwargs):
225 Get the keys available
in the repository/repositories.
228 :
return: A dict of {key:valueType}
230 # todo: getKeys is not in the mapper API
231 if self._mapper is None:
233 keys = self._mapper.getKeys(*args, **kwargs)
236 def map(self, *args, **kwargs):
237 """Find a butler location
for the given arguments.
238 See mapper.map
for more information about args
and kwargs.
240 :param args: arguments to be passed on to mapper.map
241 :param kwargs: keyword arguments to be passed on to mapper.map
242 :
return: The type of item
is dependent on the mapper being used but
is typically a ButlerLocation.
244 if self._mapper is None:
245 raise RuntimeError("No mapper assigned to Repository")
246 loc = self._mapper.map(*args, **kwargs)
249 loc.setRepository(self)
252 def queryMetadata(self, *args, **kwargs):
253 """Gets possible values
for keys given a partial data id.
255 See mapper documentation
for more explanation about queryMetadata.
257 :param args: arguments to be passed on to mapper.queryMetadata
258 :param kwargs: keyword arguments to be passed on to mapper.queryMetadata
259 :
return:The type of item
is dependent on the mapper being used but
is typically a set that contains
260 available values
for the keys
in the format input argument.
262 if self._mapper is None:
264 ret = self._mapper.queryMetadata(*args, **kwargs)
267 def backup(self, *args, **kwargs):
268 """Perform mapper.backup.
270 See mapper.backup
for more information about args
and kwargs.
272 :param args: arguments to be passed on to mapper.backup
273 :param kwargs: keyword arguments to be passed on to mapper.backup
276 if self._mapper is None:
278 self._mapper.backup(*args, **kwargs)
280 def getMapperDefaultLevel(self):
281 """Get the default level of the mapper.
283 This
is typically used
if no level
is passed into butler methods that call repository.getKeys
and/
or
284 repository.queryMetadata. There
is a bug
in that code because it gets the default level
from this
285 repository but then uses that value when searching all repositories. If this
and other repositories
286 have dissimilar data, the default level value will be nonsensical. A good example of this issue
is in
287 Butler.subset; it needs refactoring.
291 if self._mapper is None:
293 return self._mapper.getDefaultLevel()
295 def exists(self, location):
296 """Check
if location exists
in storage.
300 location : ButlerLocation
301 Desrcibes a location
in storage to look
for.
306 True if location exists,
False if not.
308 butlerLocationStorage = location.getStorage()
309 if butlerLocationStorage:
310 return butlerLocationStorage.exists(location)
312 return self._storage.exists(location)