24 from past.builtins
import basestring
37 from .
import (LogicalLocation, Persistence, Policy, StorageList,
38 StorageInterface, Storage, ButlerLocation,
39 NoRepositroyAtRoot, RepositoryCfg)
40 from lsst.log
import Log
41 import lsst.pex.policy
as pexPolicy
42 from .safeFileIo
import SafeFilename, safeMakeDir
43 from future
import standard_library
44 standard_library.install_aliases()
47 """Defines the interface for a storage location on the local filesystem.
52 URI or path that is used as the storage location.
54 If True a new repository will be created at the root location if it
55 does not exist. If False then a new repository will not be created.
60 If create is False and a repository does not exist at the root
61 specified by uri then NoRepositroyAtRoot is raised.
65 self.
log = Log.getLogger(
"daf.persistence.butler")
67 if self.
root and not os.path.exists(self.
root):
69 raise NoRepositroyAtRoot(
"No repository at {}".format(uri))
73 persistencePolicy = pexPolicy.Policy()
74 self.
persistence = Persistence.getPersistence(persistencePolicy)
77 return 'PosixStorage(root=%s)' % self.
root
80 def _pathFromURI(uri):
81 """Get the path part of the URI"""
82 return urllib.parse.urlparse(uri).path
86 """Get a relative path from a location to a location.
91 A path at which to start. It can be a relative path or an
94 A target location. It can be a relative path or an absolute path.
99 A relative path that describes the path from fromPath to toPath.
101 fromPath = os.path.realpath(fromPath)
102 return os.path.relpath(toPath, fromPath)
106 """Get an absolute path for the path from fromUri to toUri
110 fromPath : the starting location
111 A location at which to start. It can be a relative path or an
113 relativePath : the location relative to fromPath
119 Path that is an absolute path representation of fromPath +
120 relativePath, if one exists. If relativePath is absolute or if
121 fromPath is not related to relativePath then relativePath will be
124 if os.path.isabs(relativePath):
126 fromPath = os.path.realpath(fromPath)
127 return os.path.normpath(os.path.join(fromPath, relativePath))
131 """Get a persisted RepositoryCfg
135 uri : URI or path to a RepositoryCfg
140 A RepositoryCfg instance or None
142 storage = Storage.makeFromURI(uri)
143 formatter = storage._getFormatter(RepositoryCfg)
144 return formatter.read(ButlerLocation(pythonType=
None,
147 locationList=
'repositoryCfg.yaml',
156 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
157 formatter = storage._getFormatter(type(cfg))
158 formatter.write(cfg, ButlerLocation(pythonType=
None,
161 locationList=
'repositoryCfg.yaml',
170 """Get the mapper class associated with a repository root.
172 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
173 new code and repositories; they should use the Repository parentCfg mechanism.
178 The location of a persisted ReositoryCfg is (new style repos), or
179 the location where a _mapper file is (old style repos).
183 A class object or a class instance, depending on the state of the
184 mapper when the repository was created.
189 cfg = PosixStorage.getRepositoryCfg(root)
195 mapperFile =
"_mapper"
196 while not os.path.exists(os.path.join(basePath, mapperFile)):
198 if os.path.exists(os.path.join(basePath,
"_parent")):
199 basePath = os.path.join(basePath,
"_parent")
204 if mapperFile
is not None:
205 mapperFile = os.path.join(basePath, mapperFile)
208 with open(mapperFile,
"r") as f:
209 mapperName = f.readline().strip()
210 components = mapperName.split(".")
211 if len(components) <= 1:
212 raise RuntimeError(
"Unqualified mapper name %s in %s" %
213 (mapperName, mapperFile))
214 pkg = importlib.import_module(
".".join(components[:-1]))
215 return getattr(pkg, components[-1])
221 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
227 A path to the folder on the local filesystem.
232 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
235 linkpath = os.path.join(root,
'_parent')
236 if os.path.exists(linkpath):
238 return os.readlink(os.path.join(root,
'_parent'))
242 return os.path.join(root,
'_parent')
245 def write(self, butlerLocation, obj):
246 """Writes an object to a location and persistence format specified by
251 butlerLocation : ButlerLocation
252 The location & formatting for the object to be written.
253 obj : object instance
254 The object to be written.
256 self.log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
258 additionalData = butlerLocation.getAdditionalData()
259 storageName = butlerLocation.getStorageName()
260 locations = butlerLocation.getLocations()
262 pythonType = butlerLocation.getPythonType()
263 if pythonType
is not None:
264 if isinstance(pythonType, basestring):
266 pythonTypeTokenList = pythonType.split(
'.')
267 importClassString = pythonTypeTokenList.pop()
268 importClassString = importClassString.strip()
269 importPackage =
".".join(pythonTypeTokenList)
270 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
271 pythonType = getattr(importType, importClassString)
276 if hasattr(pythonType,
'butlerWrite'):
277 pythonType.butlerWrite(obj, butlerLocation=butlerLocation)
283 if storageName ==
"PickleStorage":
284 with open(logLoc.locString(),
"wb")
as outfile:
285 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
288 if storageName ==
"ConfigStorage":
289 obj.save(logLoc.locString())
292 if storageName ==
"FitsCatalogStorage":
293 if additionalData.exists(
"flags"):
294 kwds = dict(flags=additionalData.getInt(
"flags"))
297 obj.writeFits(logLoc.locString(), **kwds)
302 storage = self.persistence.getPersistStorage(storageName, logLoc)
303 storageList.append(storage)
305 if storageName ==
'FitsStorage':
306 self.persistence.persist(obj, storageList, additionalData)
310 if hasattr(obj,
'__deref__'):
312 self.persistence.persist(obj.__deref__(), storageList, additionalData)
314 self.persistence.persist(obj, storageList, additionalData)
316 def read(self, butlerLocation):
317 """Read from a butlerLocation.
321 butlerLocation : ButlerLocation
322 The location & formatting for the object(s) to be read.
326 A list of objects as described by the butler location. One item for
327 each location in butlerLocation.getLocations()
329 additionalData = butlerLocation.getAdditionalData()
331 storageName = butlerLocation.getStorageName()
333 locations = butlerLocation.getLocations()
334 pythonType = butlerLocation.getPythonType()
335 if pythonType
is not None:
336 if isinstance(pythonType, basestring):
338 pythonTypeTokenList = pythonType.split(
'.')
339 importClassString = pythonTypeTokenList.pop()
340 importClassString = importClassString.strip()
341 importPackage =
".".join(pythonTypeTokenList)
342 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
343 pythonType = getattr(importType, importClassString)
347 if hasattr(pythonType,
'butlerRead'):
348 results = pythonType.butlerRead(butlerLocation=butlerLocation)
351 for locationString
in locations:
352 locationString = os.path.join(self.
root, locationString)
356 if storageName ==
"PafStorage":
357 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
358 elif storageName ==
"YamlStorage":
359 finalItem = Policy(filePath=logLoc.locString())
360 elif storageName ==
"PickleStorage":
361 if not os.path.exists(logLoc.locString()):
362 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
363 with open(logLoc.locString(),
"rb")
as infile:
367 if sys.version_info.major >= 3:
368 finalItem = pickle.load(infile, encoding=
"latin1")
370 finalItem = pickle.load(infile)
371 elif storageName ==
"FitsCatalogStorage":
372 if not os.path.exists(logLoc.locString()):
373 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
375 if additionalData.exists(
"hdu"):
376 kwds[
"hdu"] = additionalData.getInt(
"hdu")
377 if additionalData.exists(
"flags"):
378 kwds[
"flags"] = additionalData.getInt(
"flags")
379 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
380 elif storageName ==
"ConfigStorage":
381 if not os.path.exists(logLoc.locString()):
382 raise RuntimeError(
"No such config file: " + logLoc.locString())
383 finalItem = pythonType()
384 finalItem.load(logLoc.locString())
387 storage = self.persistence.getRetrieveStorage(storageName, logLoc)
388 storageList.append(storage)
389 finalItem = self.persistence.unsafeRetrieve(
390 butlerLocation.getCppType(), storageList, additionalData)
391 results.append(finalItem)
396 """Implementaion of PosixStorage.exists for ButlerLocation objects."""
397 storageName = location.getStorageName()
398 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
399 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage'):
400 self.log.warn(
"butlerLocationExists for non-supported storage %s" % location)
402 for locationString
in location.getLocations():
403 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
410 """Check if location exists.
414 location : ButlerLocation or string
415 A a string or a ButlerLocation that describes the location of an
416 object in this storage.
421 True if exists, else False.
423 if isinstance(location, ButlerLocation):
430 """Get the full path to the location.
435 return os.path.join(self.
root, location)
439 """Test if a Version 1 Repository exists.
441 Version 1 Repositories only exist in posix storages, do not have a
442 RepositoryCfg file, and contain either a registry.sqlite3 file, a
443 _mapper file, or a _parent link.
448 A path to a folder on the local filesystem.
453 True if the repository at root exists, else False.
455 return os.path.exists(root)
and (
456 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or
457 os.path.exists(os.path.join(root,
"_mapper"))
or
458 os.path.exists(os.path.join(root,
"_parent"))
462 """Copy a file from one location to another on the local filesystem.
467 Path and name of existing file.
469 Path and name of new file.
475 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
478 """Get a handle to a local copy of the file, downloading it to a
483 A path the the file in storage, relative to root.
487 A handle to a local copy of the file. If storage is remote it will be
488 a temporary file. If storage is local it may be the original file or
489 a temporary file. The file name can be gotten via the 'name' property
490 of the returned object.
492 p = os.path.join(self.
root, path)
502 """Search for the given path in this storage instance.
504 If the path contains an HDU indicator (a number in brackets before the
505 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
506 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
507 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
512 A filename (and optionally prefix path) to search for within root.
517 The location that was found, or None if no location was found.
522 def search(root, path, searchParents=False):
523 """Look for the given path in the current root.
525 Also supports searching for the path in Butler v1 repositories by
526 following the Butler v1 _parent symlink
528 If the path contains an HDU indicator (a number in brackets, e.g.
529 'foo.fits[1]', this will be stripped when searching and so
530 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
531 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
536 The path to the root directory.
538 The path to the file within the root directory.
539 searchParents : bool, optional
540 For Butler v1 repositories only, if true and a _parent symlink
541 exists, then the directory at _parent will be searched if the file
542 is not found in the root repository. Will continue searching the
543 parent of the parent until the file is found or no additional
549 The location that was found, or None if no location was found.
555 while len(rootDir) > 1
and rootDir[-1] ==
'/':
556 rootDir = rootDir[:-1]
558 if path.startswith(rootDir +
"/"):
560 path = path[len(rootDir +
'/'):]
562 elif rootDir ==
"/" and path.startswith(
"/"):
567 pathPrefix = os.path.dirname(path)
568 while pathPrefix !=
"" and pathPrefix !=
"/":
569 if os.path.realpath(pathPrefix) == os.path.realpath(root):
571 pathPrefix = os.path.dirname(pathPrefix)
572 if pathPrefix ==
"/":
574 elif pathPrefix !=
"":
575 path = path[len(pathPrefix)+1:]
581 firstBracket = path.find(
"[")
582 if firstBracket != -1:
583 strippedPath = path[:firstBracket]
584 pathStripped = path[firstBracket:]
588 paths = glob.glob(os.path.join(dir, strippedPath))
590 if pathPrefix != rootDir:
591 paths = [p[len(rootDir+
'/'):]
for p
in paths]
592 if pathStripped
is not None:
593 paths = [p + pathStripped
for p
in paths]
596 dir = os.path.join(dir,
"_parent")
597 if not os.path.exists(dir):
604 """Ask if a storage at the location described by uri exists
609 URI to the the root location of the storage
614 True if the storage exists, false if not
616 return os.path.exists(PosixStorage._pathFromURI(uri))
618 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
619 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
Class for logical location of a persisted Persistable instance.