24 from past.builtins
import basestring
25 from builtins
import object
36 """Arguments passed into a Butler that are used to instantiate a repository. This includes arguments that
37 can be used to create a new repository (cfgRoot, root, mapper, mapperArgs, policy) and are persisted along
38 with the new repository's configuration file. These arguments can also describe how a new or existing
39 repository are to be used (cfgRoot or root, tags, mode). When indicating an existing repository it is
40 better to not specify unnecessary arguments, as if they conflict with the persisted repository
41 configuration then a RuntimeError will be raised during Butler init.
43 A RepositoryArgs class can be initialized from a dict, if the first argument to the initializer is a dict.
47 cfgRoot : URI or dict, optional
48 If dict, the initalizer is re-called with the expanded dict.
49 If URI, this is the location where the RepositoryCfg should be found (existing repo) or put (new repo)
51 If different than cfgRoot then this is the location where the repository should exist. A RepositoryCfg
52 will be put at cfgRoot and its root will be a path to root.
53 mapper : string or class object, optional
54 The mapper to use with this repository. If string, should refer an importable object. If class object,
55 should be a mapper to be instantiated by the Butler during Butler init.
56 tags : list or object, optional
57 One or more unique identifiers to uniquely identify this repository and its parents when performing
59 mode : string, optional
60 should be one of 'r', 'w', or 'rw', for 'read', 'write', or 'read-write'. Can be omitted; input
61 repositories will default to '
r', output repositories will default to 'w'. 'w' on an input repository
62 will raise a RuntimeError during Butler init, although 'rw' works and is equivalent to '
r'. Output
63 repositories may be 'r' or 'rw', '
r' for an output repository will raise a RuntimeError during Butler
66 def __init__(self, cfgRoot=None, root=None, mapper=None, mapperArgs=None, tags=None,
67 mode=None, policy=None):
69 # is cfgRoot a dict? try dict init:
70 self.__init__(**cfgRoot)
72 self._root = Storage.absolutePath(os.getcwd(), root.rstrip(os.sep)) if root else root
73 self._cfgRoot = Storage.absolutePath(os.getcwd(), cfgRoot.rstrip(os.sep)) if cfgRoot else cfgRoot
75 self.mapperArgs = mapperArgs
76 self.tags = set(listify(tags))
78 self.policy = Policy(policy) if policy is not None else None
81 return "%s(root=%r, cfgRoot=%r, mapper=%r, mapperArgs=%r, tags=%s, mode=%r, policy=%s)" % (
82 self.__class__.__name__, self.root, self._cfgRoot, self._mapper, self.mapperArgs, self.tags,
83 self.mode, self.policy)
90 def mapper(self, mapper):
91 if mapper is not None and self._mapper:
92 raise RuntimeError("Explicity clear mapper (set to None) before changing its value.")
97 return self._cfgRoot if self._cfgRoot is not None else self._root
101 return self._root if self._root is not None else self._cfgRoot
104 def inputRepo(storage, tags=None):
105 return RepositoryArgs(storage, tags)
108 def outputRepo(storage, mapper=None, mapperArgs=None, tags=None, mode=None):
109 return RepositoryArgs(storage, mapper, mapperArgs, tags, mode)
112 """add a tag to the repository cfg
"""
113 if isinstance(tag, basestring):
117 self.tags.update(tag)
122 class Repository(object):
123 """Represents a repository of persisted data
and has methods to access that data.
126 def __init__(self, repoData):
127 """Initialize a Repository with parameters input via RepoData.
132 Object that contains the parameters with which to init the Repository.
134 self._storage = Storage.makeFromURI(repoData.cfg.root)
135 if repoData.isNewRepository and not repoData.isV1Repository:
136 self._storage.putRepositoryCfg(repoData.cfg, repoData.args.cfgRoot)
137 self._mapperArgs = repoData.cfg.mapperArgs # keep for reference in matchesArgs
138 self._initMapper(repoData)
140 def _initMapper(self, repoData):
141 '''Initialize and keep the mapper in a member var.
146 The RepoData with the properties of this Repository.
149 # rule: If mapper is:
150 # - an object: use it as the mapper.
151 # - a string: import it and instantiate it with mapperArgs
152 # - a class object: instantiate it with mapperArgs
153 mapper = repoData.cfg.mapper
155 # if mapper is a string, import it:
156 if isinstance(mapper, basestring):
157 mapper = doImport(mapper)
158 # now if mapper is a class type (not instance), instantiate it:
159 if inspect.isclass(mapper):
160 mapperArgs = copy.copy(repoData.cfg.mapperArgs)
161 if mapperArgs is None:
163 if 'root' not in mapperArgs:
164 mapperArgs['root'] = repoData.cfg.root
165 mapper = mapper(parentRegistry=repoData.parentRegistry,
166 repositoryCfg=repoData.cfg,
168 self._mapper = mapper
171 return 'config(id=%s, storage=%s, parent=%s, mapper=%s, mapperArgs=%s, cls=%s)' % \
172 (self.id, self._storage, self.parent, self._mapper, self.mapperArgs, self.cls)
174 # todo want a way to make a repository read-only
175 def write(self, butlerLocation, obj):
176 """Write a dataset to Storage.
178 :param butlerLocation: Contains the details needed to find the desired dataset.
179 :param dataset: The dataset to be written.
182 butlerLocationStorage = butlerLocation.getStorage()
183 if butlerLocationStorage:
184 return butlerLocationStorage.write(butlerLocation, obj)
186 return self._storage.write(butlerLocation, obj)
188 def read(self, butlerLocation):
189 """Read a dataset
from Storage.
191 :param butlerLocation: Contains the details needed to find the desired dataset.
192 :
return: An instance of the dataset requested by butlerLocation.
194 butlerLocationStorage = butlerLocation.getStorage()
195 if butlerLocationStorage:
196 return butlerLocationStorage.read(butlerLocation)
198 return self._storage.read(butlerLocation)
204 return (self._mapper, )
206 def getRegistry(self):
207 """Get the registry
from the mapper
212 The registry
from the mapper
or None if the mapper does
not have one.
214 if self._mapper is None:
216 return self._mapper.getRegistry()
218 def getKeys(self, *args, **kwargs):
220 Get the keys available
in the repository/repositories.
223 :
return: A dict of {key:valueType}
225 # todo: getKeys is not in the mapper API
226 if self._mapper is None:
228 keys = self._mapper.getKeys(*args, **kwargs)
231 def map(self, *args, **kwargs):
232 """Find a butler location
for the given arguments.
233 See mapper.map
for more information about args
and kwargs.
235 :param args: arguments to be passed on to mapper.map
236 :param kwargs: keyword arguments to be passed on to mapper.map
237 :
return: The type of item
is dependent on the mapper being used but
is typically a ButlerLocation.
239 if self._mapper is None:
240 raise RuntimeError("No mapper assigned to Repository")
241 loc = self._mapper.map(*args, **kwargs)
244 loc.setRepository(self)
247 def queryMetadata(self, *args, **kwargs):
248 """Gets possible values
for keys given a partial data id.
250 See mapper documentation
for more explanation about queryMetadata.
252 :param args: arguments to be passed on to mapper.queryMetadata
253 :param kwargs: keyword arguments to be passed on to mapper.queryMetadata
254 :
return:The type of item
is dependent on the mapper being used but
is typically a set that contains
255 available values
for the keys
in the format input argument.
257 if self._mapper is None:
259 ret = self._mapper.queryMetadata(*args, **kwargs)
262 def backup(self, *args, **kwargs):
263 """Perform mapper.backup.
265 See mapper.backup
for more information about args
and kwargs.
267 :param args: arguments to be passed on to mapper.backup
268 :param kwargs: keyword arguments to be passed on to mapper.backup
271 if self._mapper is None:
273 self._mapper.backup(*args, **kwargs)
275 def getMapperDefaultLevel(self):
276 """Get the default level of the mapper.
278 This
is typically used
if no level
is passed into butler methods that call repository.getKeys
and/
or
279 repository.queryMetadata. There
is a bug
in that code because it gets the default level
from this
280 repository but then uses that value when searching all repositories. If this
and other repositories
281 have dissimilar data, the default level value will be nonsensical. A good example of this issue
is in
282 Butler.subset; it needs refactoring.
286 if self._mapper is None:
288 return self._mapper.getDefaultLevel()
290 def exists(self, location):
291 """Check
if location exists
in storage.
295 location : ButlerLocation
296 Desrcibes a location
in storage to look
for.
301 True if location exists,
False if not.
303 butlerLocationStorage = location.getStorage()
304 if butlerLocationStorage:
305 return butlerLocationStorage.exists(location)
307 return self._storage.exists(location)