24 from past.builtins
import basestring
36 from .
import (LogicalLocation, Persistence, Policy, StorageList,
37 StorageInterface, Storage, safeFileIo, ButlerLocation,
39 from lsst.log
import Log
40 import lsst.pex.policy
as pexPolicy
41 from .safeFileIo
import SafeFilename, safeMakeDir
42 from future
import standard_library
43 standard_library.install_aliases()
47 """Defines the interface for a storage location on the local filesystem.
52 URI or path that is used as the storage location.
54 If True a new repository will be created at the root location if it
55 does not exist. If False then a new repository will not be created.
60 If create is False and a repository does not exist at the root
61 specified by uri then NoRepositroyAtRoot is raised.
65 self.
log = Log.getLogger(
"daf.persistence.butler")
67 if self.
root and not os.path.exists(self.
root):
69 raise NoRepositroyAtRoot(
"No repository at {}".format(uri))
73 persistencePolicy = pexPolicy.Policy()
74 self.
persistence = Persistence.getPersistence(persistencePolicy)
77 return 'PosixStorage(root=%s)' % self.
root
80 def _pathFromURI(uri):
81 """Get the path part of the URI"""
82 return urllib.parse.urlparse(uri).path
86 """Get a relative path from a location to a location.
91 A path at which to start. It can be a relative path or an
94 A target location. It can be a relative path or an absolute path.
99 A relative path that describes the path from fromPath to toPath.
101 fromPath = os.path.realpath(fromPath)
102 return os.path.relpath(toPath, fromPath)
106 """Get an absolute path for the path from fromUri to toUri
110 fromPath : the starting location
111 A location at which to start. It can be a relative path or an
113 relativePath : the location relative to fromPath
119 Path that is an absolute path representation of fromPath +
120 relativePath, if one exists. If relativePath is absolute or if
121 fromPath is not related to relativePath then relativePath will be
124 if os.path.isabs(relativePath):
126 fromPath = os.path.realpath(fromPath)
127 return os.path.normpath(os.path.join(fromPath, relativePath))
130 def _getRepositoryCfg(uri):
131 """Get a persisted RepositoryCfg
135 uri : URI or path to a RepositoryCfg
140 A RepositoryCfg instance or None
143 parseRes = urllib.parse.urlparse(uri)
144 loc = os.path.join(parseRes.path,
'repositoryCfg.yaml')
145 if os.path.exists(loc):
146 with open(loc,
'r') as f:
147 repositoryCfg = yaml.load(f)
148 if repositoryCfg.root
is None:
149 repositoryCfg.root = uri
154 """Get a persisted RepositoryCfg
158 uri : URI or path to a RepositoryCfg
163 A RepositoryCfg instance or None
165 repositoryCfg = PosixStorage._getRepositoryCfg(uri)
166 if repositoryCfg
is not None:
173 """Serialize a RepositoryCfg to a location.
175 When loc == cfg.root, the RepositoryCfg is to be writtenat the root
176 location of the repository. In that case, root is not written, it is
177 implicit in the location of the cfg. This allows the cfg to move from
178 machine to machine without modification.
182 cfg : RepositoryCfg instance
183 The RepositoryCfg to be serailized.
185 The location to write the RepositoryCfg. If loc is None, the
186 location will be read from the root parameter of loc.
192 if loc
is None or cfg.root == loc:
198 parseRes = urllib.parse.urlparse(loc)
200 if not os.path.exists(loc):
202 loc = os.path.join(loc,
'repositoryCfg.yaml')
203 with safeFileIo.FileForWriteOnceCompareSame(loc)
as f:
208 """Get the mapper class associated with a repository root.
210 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
211 new code and repositories; they should use the Repository parentCfg mechanism.
216 The location of a persisted ReositoryCfg is (new style repos), or
217 the location where a _mapper file is (old style repos).
221 A class object or a class instance, depending on the state of the
222 mapper when the repository was created.
227 cfg = PosixStorage._getRepositoryCfg(root)
233 mapperFile =
"_mapper"
234 while not os.path.exists(os.path.join(basePath, mapperFile)):
236 if os.path.exists(os.path.join(basePath,
"_parent")):
237 basePath = os.path.join(basePath,
"_parent")
242 if mapperFile
is not None:
243 mapperFile = os.path.join(basePath, mapperFile)
246 with open(mapperFile,
"r") as f:
247 mapperName = f.readline().strip()
248 components = mapperName.split(".")
249 if len(components) <= 1:
250 raise RuntimeError(
"Unqualified mapper name %s in %s" %
251 (mapperName, mapperFile))
252 pkg = importlib.import_module(
".".join(components[:-1]))
253 return getattr(pkg, components[-1])
259 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
265 A path to the folder on the local filesystem.
270 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
273 linkpath = os.path.join(root,
'_parent')
274 if os.path.exists(linkpath):
276 return os.readlink(os.path.join(root,
'_parent'))
280 return os.path.join(root,
'_parent')
283 def write(self, butlerLocation, obj):
284 """Writes an object to a location and persistence format specified by
289 butlerLocation : ButlerLocation
290 The location & formatting for the object to be written.
291 obj : object instance
292 The object to be written.
294 self.log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
296 additionalData = butlerLocation.getAdditionalData()
297 storageName = butlerLocation.getStorageName()
298 locations = butlerLocation.getLocations()
300 pythonType = butlerLocation.getPythonType()
301 if pythonType
is not None:
302 if isinstance(pythonType, basestring):
304 pythonTypeTokenList = pythonType.split(
'.')
305 importClassString = pythonTypeTokenList.pop()
306 importClassString = importClassString.strip()
307 importPackage =
".".join(pythonTypeTokenList)
308 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
309 pythonType = getattr(importType, importClassString)
314 if hasattr(pythonType,
'butlerWrite'):
315 pythonType.butlerWrite(obj, butlerLocation=butlerLocation)
321 if storageName ==
"PickleStorage":
322 with open(logLoc.locString(),
"wb")
as outfile:
323 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
326 if storageName ==
"ConfigStorage":
327 obj.save(logLoc.locString())
330 if storageName ==
"FitsCatalogStorage":
331 flags = additionalData.getInt(
"flags", 0)
332 obj.writeFits(logLoc.locString(), flags=flags)
337 storage = self.persistence.getPersistStorage(storageName, logLoc)
338 storageList.append(storage)
340 if storageName ==
'FitsStorage':
341 self.persistence.persist(obj, storageList, additionalData)
345 if hasattr(obj,
'__deref__'):
347 self.persistence.persist(obj.__deref__(), storageList, additionalData)
349 self.persistence.persist(obj, storageList, additionalData)
351 def read(self, butlerLocation):
352 """Read from a butlerLocation.
356 butlerLocation : ButlerLocation
357 The location & formatting for the object(s) to be read.
361 A list of objects as described by the butler location. One item for
362 each location in butlerLocation.getLocations()
364 additionalData = butlerLocation.getAdditionalData()
366 storageName = butlerLocation.getStorageName()
368 locations = butlerLocation.getLocations()
369 pythonType = butlerLocation.getPythonType()
370 if pythonType
is not None:
371 if isinstance(pythonType, basestring):
373 pythonTypeTokenList = pythonType.split(
'.')
374 importClassString = pythonTypeTokenList.pop()
375 importClassString = importClassString.strip()
376 importPackage =
".".join(pythonTypeTokenList)
377 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
378 pythonType = getattr(importType, importClassString)
382 if hasattr(pythonType,
'butlerRead'):
383 results = pythonType.butlerRead(butlerLocation=butlerLocation)
386 for locationString
in locations:
387 locationString = os.path.join(self.
root, locationString)
391 if storageName ==
"PafStorage":
392 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
393 elif storageName ==
"YamlStorage":
394 finalItem = Policy(filePath=logLoc.locString())
395 elif storageName ==
"PickleStorage":
396 if not os.path.exists(logLoc.locString()):
397 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
398 with open(logLoc.locString(),
"rb")
as infile:
402 if sys.version_info.major >= 3:
403 finalItem = pickle.load(infile, encoding=
"latin1")
405 finalItem = pickle.load(infile)
406 elif storageName ==
"FitsCatalogStorage":
407 if not os.path.exists(logLoc.locString()):
408 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
410 hdu = additionalData.getInt(
"hdu", INT_MIN)
411 flags = additionalData.getInt(
"flags", 0)
412 finalItem = pythonType.readFits(logLoc.locString(), hdu, flags)
413 elif storageName ==
"ConfigStorage":
414 if not os.path.exists(logLoc.locString()):
415 raise RuntimeError(
"No such config file: " + logLoc.locString())
416 finalItem = pythonType()
417 finalItem.load(logLoc.locString())
420 storage = self.persistence.getRetrieveStorage(storageName, logLoc)
421 storageList.append(storage)
422 finalItem = self.persistence.unsafeRetrieve(
423 butlerLocation.getCppType(), storageList, additionalData)
424 results.append(finalItem)
429 """Implementaion of PosixStorage.exists for ButlerLocation objects."""
430 storageName = location.getStorageName()
431 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
432 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage'):
433 self.log.warn(
"butlerLocationExists for non-supported storage %s" % location)
435 for locationString
in location.getLocations():
436 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
443 """Check if location exists.
447 location : ButlerLocation or string
448 A a string or a ButlerLocation that describes the location of an
449 object in this storage.
454 True if exists, else False.
456 if isinstance(location, ButlerLocation):
463 """Get the full path to the location.
468 return os.path.join(self.
root, location)
472 """Test if a Version 1 Repository exists.
474 Version 1 Repositories only exist in posix storages and do not have a RepositoryCfg file.
475 To "exist" the folder at root must exist and contain files or folders.
480 A path to a folder on the local filesystem.
485 True if the repository at root exists, else False.
487 return os.path.exists(root)
and bool(os.listdir(root))
490 """Copy a file from one location to another on the local filesystem.
495 Path and name of existing file.
497 Path and name of new file.
503 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
506 """Get a handle to a local copy of the file, downloading it to a
511 A path the the file in storage, relative to root.
515 A handle to a local copy of the file. If storage is remote it will be
516 a temporary file. If storage is local it may be the original file or
517 a temporary file. The file name can be gotten via the 'name' property
518 of the returned object.
520 p = os.path.join(self.
root, path)
530 """Search for the given path in this storage instance.
532 If the path contains an HDU indicator (a number in brackets before the
533 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
534 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
535 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
540 A filename (and optionally prefix path) to search for within root.
545 The location that was found, or None if no location was found.
550 def search(root, path, searchParents=False):
551 """Look for the given path in the current root.
553 Also supports searching for the path in Butler v1 repositories by
554 following the Butler v1 _parent symlink
556 If the path contains an HDU indicator (a number in brackets, e.g.
557 'foo.fits[1]', this will be stripped when searching and so
558 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
559 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
564 The path to the root directory.
566 The path to the file within the root directory.
567 searchParents : bool, optional
568 For Butler v1 repositories only, if true and a _parent symlink
569 exists, then the directory at _parent will be searched if the file
570 is not found in the root repository. Will continue searching the
571 parent of the parent until the file is found or no additional
577 The location that was found, or None if no location was found.
583 while len(rootDir) > 1
and rootDir[-1] ==
'/':
584 rootDir = rootDir[:-1]
586 if path.startswith(rootDir +
"/"):
588 path = path[len(rootDir +
'/'):]
590 elif rootDir ==
"/" and path.startswith(
"/"):
595 pathPrefix = os.path.dirname(path)
596 while pathPrefix !=
"" and pathPrefix !=
"/":
597 if os.path.realpath(pathPrefix) == os.path.realpath(root):
599 pathPrefix = os.path.dirname(pathPrefix)
600 if pathPrefix ==
"/":
602 elif pathPrefix !=
"":
603 path = path[len(pathPrefix)+1:]
609 firstBracket = path.find(
"[")
610 if firstBracket != -1:
611 strippedPath = path[:firstBracket]
612 pathStripped = path[firstBracket:]
616 paths = glob.glob(os.path.join(dir, strippedPath))
618 if pathPrefix != rootDir:
619 paths = [p[len(rootDir+
'/'):]
for p
in paths]
620 if pathStripped
is not None:
621 paths = [p + pathStripped
for p
in paths]
624 dir = os.path.join(dir,
"_parent")
625 if not os.path.exists(dir):
632 """Ask if a storage at the location described by uri exists
637 URI to the the root location of the storage
642 True if the storage exists, false if not
644 return os.path.exists(PosixStorage._pathFromURI(uri))
647 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
648 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
Class for logical location of a persisted Persistable instance.