24 from past.builtins
import basestring
37 from .
import (LogicalLocation, Persistence, Policy, StorageList,
38 StorageInterface, Storage, ButlerLocation,
39 NoRepositroyAtRoot, RepositoryCfg)
40 from lsst.log
import Log
41 import lsst.pex.policy
as pexPolicy
42 from .safeFileIo
import SafeFilename, safeMakeDir
43 from future
import standard_library
44 standard_library.install_aliases()
47 """Defines the interface for a storage location on the local filesystem. 52 URI or path that is used as the storage location. 54 If True a new repository will be created at the root location if it 55 does not exist. If False then a new repository will not be created. 60 If create is False and a repository does not exist at the root 61 specified by uri then NoRepositroyAtRoot is raised. 65 self.
log = Log.getLogger(
"daf.persistence.butler")
67 if self.
root and not os.path.exists(self.
root):
73 persistencePolicy = pexPolicy.Policy()
74 self.
persistence = Persistence.getPersistence(persistencePolicy)
77 return 'PosixStorage(root=%s)' % self.
root 80 def _pathFromURI(uri):
81 """Get the path part of the URI""" 82 return urllib.parse.urlparse(uri).path
86 """Get a relative path from a location to a location. 91 A path at which to start. It can be a relative path or an 94 A target location. It can be a relative path or an absolute path. 99 A relative path that describes the path from fromPath to toPath. 101 fromPath = os.path.realpath(fromPath)
102 return os.path.relpath(toPath, fromPath)
106 """Get an absolute path for the path from fromUri to toUri 110 fromPath : the starting location 111 A location at which to start. It can be a relative path or an 113 relativePath : the location relative to fromPath 119 Path that is an absolute path representation of fromPath + 120 relativePath, if one exists. If relativePath is absolute or if 121 fromPath is not related to relativePath then relativePath will be 124 if os.path.isabs(relativePath):
126 fromPath = os.path.realpath(fromPath)
127 return os.path.normpath(os.path.join(fromPath, relativePath))
131 """Get a persisted RepositoryCfg 135 uri : URI or path to a RepositoryCfg 140 A RepositoryCfg instance or None 142 storage = Storage.makeFromURI(uri)
143 formatter = storage._getFormatter(RepositoryCfg)
147 locationList=
'repositoryCfg.yaml',
156 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
157 formatter = storage._getFormatter(type(cfg))
161 locationList=
'repositoryCfg.yaml',
170 """Get the mapper class associated with a repository root. 172 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 173 new code and repositories; they should use the Repository parentCfg mechanism. 178 The location of a persisted ReositoryCfg is (new style repos), or 179 the location where a _mapper file is (old style repos). 183 A class object or a class instance, depending on the state of the 184 mapper when the repository was created. 189 cfg = PosixStorage.getRepositoryCfg(root)
195 mapperFile =
"_mapper" 196 while not os.path.exists(os.path.join(basePath, mapperFile)):
198 if os.path.exists(os.path.join(basePath,
"_parent")):
199 basePath = os.path.join(basePath,
"_parent")
204 if mapperFile
is not None:
205 mapperFile = os.path.join(basePath, mapperFile)
208 with open(mapperFile,
"r") as f: 209 mapperName = f.readline().strip() 210 components = mapperName.split(".")
211 if len(components) <= 1:
212 raise RuntimeError(
"Unqualified mapper name %s in %s" %
213 (mapperName, mapperFile))
214 pkg = importlib.import_module(
".".join(components[:-1]))
215 return getattr(pkg, components[-1])
221 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 227 A path to the folder on the local filesystem. 232 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 235 linkpath = os.path.join(root,
'_parent')
236 if os.path.exists(linkpath):
238 return os.readlink(os.path.join(root,
'_parent'))
242 return os.path.join(root,
'_parent')
245 def write(self, butlerLocation, obj):
246 """Writes an object to a location and persistence format specified by 251 butlerLocation : ButlerLocation 252 The location & formatting for the object to be written. 253 obj : object instance 254 The object to be written. 256 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
258 additionalData = butlerLocation.getAdditionalData()
259 storageName = butlerLocation.getStorageName()
260 locations = butlerLocation.getLocations()
262 pythonType = butlerLocation.getPythonType()
263 if pythonType
is not None:
264 if isinstance(pythonType, basestring):
266 pythonTypeTokenList = pythonType.split(
'.')
267 importClassString = pythonTypeTokenList.pop()
268 importClassString = importClassString.strip()
269 importPackage =
".".join(pythonTypeTokenList)
270 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
271 pythonType = getattr(importType, importClassString)
276 if hasattr(pythonType,
'butlerWrite'):
277 pythonType.butlerWrite(obj, butlerLocation=butlerLocation)
283 if storageName ==
"PickleStorage":
284 with open(logLoc.locString(),
"wb")
as outfile:
285 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
288 if storageName ==
"ConfigStorage":
289 obj.save(logLoc.locString())
292 if storageName ==
"FitsCatalogStorage":
293 if additionalData.exists(
"flags"):
294 kwds = dict(flags=additionalData.getInt(
"flags"))
297 obj.writeFits(logLoc.locString(), **kwds)
302 storage = self.
persistence.getPersistStorage(storageName, logLoc)
303 storageList.append(storage)
305 if storageName ==
'FitsStorage':
306 self.
persistence.persist(obj, storageList, additionalData)
310 if hasattr(obj,
'__deref__'):
312 self.
persistence.persist(obj.__deref__(), storageList, additionalData)
314 self.
persistence.persist(obj, storageList, additionalData)
316 def read(self, butlerLocation):
317 """Read from a butlerLocation. 321 butlerLocation : ButlerLocation 322 The location & formatting for the object(s) to be read. 326 A list of objects as described by the butler location. One item for 327 each location in butlerLocation.getLocations() 329 additionalData = butlerLocation.getAdditionalData()
331 storageName = butlerLocation.getStorageName()
333 locations = butlerLocation.getLocations()
334 pythonType = butlerLocation.getPythonType()
335 if pythonType
is not None:
336 if isinstance(pythonType, basestring):
338 pythonTypeTokenList = pythonType.split(
'.')
339 importClassString = pythonTypeTokenList.pop()
340 importClassString = importClassString.strip()
341 importPackage =
".".join(pythonTypeTokenList)
342 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
343 pythonType = getattr(importType, importClassString)
347 if hasattr(pythonType,
'butlerRead'):
348 results = pythonType.butlerRead(butlerLocation=butlerLocation)
351 for locationString
in locations:
352 locationString = os.path.join(self.
root, locationString)
356 if storageName ==
"PafStorage":
357 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
358 elif storageName ==
"YamlStorage":
359 finalItem =
Policy(filePath=logLoc.locString())
360 elif storageName ==
"PickleStorage":
361 if not os.path.exists(logLoc.locString()):
362 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
363 with open(logLoc.locString(),
"rb")
as infile:
367 if sys.version_info.major >= 3:
368 finalItem = pickle.load(infile, encoding=
"latin1")
370 finalItem = pickle.load(infile)
371 elif storageName ==
"FitsCatalogStorage":
372 if not os.path.exists(logLoc.locString()):
373 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
375 if additionalData.exists(
"hdu"):
376 kwds[
"hdu"] = additionalData.getInt(
"hdu")
377 if additionalData.exists(
"flags"):
378 kwds[
"flags"] = additionalData.getInt(
"flags")
379 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
380 elif storageName ==
"ConfigStorage":
381 if not os.path.exists(logLoc.locString()):
382 raise RuntimeError(
"No such config file: " + logLoc.locString())
383 finalItem = pythonType()
384 finalItem.load(logLoc.locString())
387 storage = self.
persistence.getRetrieveStorage(storageName, logLoc)
388 storageList.append(storage)
390 butlerLocation.getCppType(), storageList, additionalData)
391 results.append(finalItem)
396 """Implementaion of PosixStorage.exists for ButlerLocation objects.""" 397 storageName = location.getStorageName()
398 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
399 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage'):
400 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
402 for locationString
in location.getLocations():
403 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
410 """Check if location exists. 414 location : ButlerLocation or string 415 A a string or a ButlerLocation that describes the location of an 416 object in this storage. 421 True if exists, else False. 423 if isinstance(location, ButlerLocation):
430 """Get the full path to the location. 435 return os.path.join(self.
root, location)
439 """Test if a Version 1 Repository exists. 441 Version 1 Repositories only exist in posix storages, do not have a 442 RepositoryCfg file, and contain either a registry.sqlite3 file, a 443 _mapper file, or a _parent link. 448 A path to a folder on the local filesystem. 453 True if the repository at root exists, else False. 455 return os.path.exists(root)
and (
456 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or 457 os.path.exists(os.path.join(root,
"_mapper"))
or 458 os.path.exists(os.path.join(root,
"_parent"))
462 """Copy a file from one location to another on the local filesystem. 467 Path and name of existing file. 469 Path and name of new file. 475 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
478 """Get a handle to a local copy of the file, downloading it to a 483 A path the the file in storage, relative to root. 487 A handle to a local copy of the file. If storage is remote it will be 488 a temporary file. If storage is local it may be the original file or 489 a temporary file. The file name can be gotten via the 'name' property 490 of the returned object. 492 p = os.path.join(self.
root, path)
502 """Search for the given path in this storage instance. 504 If the path contains an HDU indicator (a number in brackets before the 505 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 506 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 507 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 512 A filename (and optionally prefix path) to search for within root. 517 The location that was found, or None if no location was found. 522 def search(root, path, searchParents=False):
523 """Look for the given path in the current root. 525 Also supports searching for the path in Butler v1 repositories by 526 following the Butler v1 _parent symlink 528 If the path contains an HDU indicator (a number in brackets, e.g. 529 'foo.fits[1]', this will be stripped when searching and so 530 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 531 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 536 The path to the root directory. 538 The path to the file within the root directory. 539 searchParents : bool, optional 540 For Butler v1 repositories only, if true and a _parent symlink 541 exists, then the directory at _parent will be searched if the file 542 is not found in the root repository. Will continue searching the 543 parent of the parent until the file is found or no additional 549 The location that was found, or None if no location was found. 555 while len(rootDir) > 1
and rootDir[-1] ==
'/':
556 rootDir = rootDir[:-1]
558 if path.startswith(rootDir +
"/"):
560 path = path[len(rootDir +
'/'):]
562 elif rootDir ==
"/" and path.startswith(
"/"):
567 pathPrefix = os.path.dirname(path)
568 while pathPrefix !=
"" and pathPrefix !=
"/":
569 if os.path.realpath(pathPrefix) == os.path.realpath(root):
571 pathPrefix = os.path.dirname(pathPrefix)
572 if pathPrefix ==
"/":
574 elif pathPrefix !=
"":
575 path = path[len(pathPrefix)+1:]
581 firstBracket = path.find(
"[")
582 if firstBracket != -1:
583 strippedPath = path[:firstBracket]
584 pathStripped = path[firstBracket:]
588 paths = glob.glob(os.path.join(dir, strippedPath))
590 if pathPrefix != rootDir:
591 paths = [p[len(rootDir+
'/'):]
for p
in paths]
592 if pathStripped
is not None:
593 paths = [p + pathStripped
for p
in paths]
596 dir = os.path.join(dir,
"_parent")
597 if not os.path.exists(dir):
604 """Ask if a storage at the location described by uri exists 609 URI to the the root location of the storage 614 True if the storage exists, false if not 616 return os.path.exists(PosixStorage._pathFromURI(uri))
618 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
619 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def copyFile(self, fromLocation, toLocation)
def safeMakeDir(directory)
Class for logical location of a persisted Persistable instance.
def relativePath(fromPath, toPath)
def exists(self, location)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def locationWithRoot(self, location)
def getParentSymlinkPath(root)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getLocalFile(self, path)
def read(self, butlerLocation)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)