24 from future
import standard_library
25 standard_library.install_aliases()
26 from past.builtins
import basestring
38 from .
import (LogicalLocation, Persistence, Policy, StorageList, Registry,
39 Storage, RepositoryCfg, safeFileIo, ButlerLocation)
40 from lsst.log
import Log
41 import lsst.pex.policy
as pexPolicy
42 from .safeFileIo
import SafeFilename, safeMakeDir
52 self.
log = Log.getLogger(
"daf.persistence.butler")
54 if self.
root and not os.path.exists(self.
root):
58 persistencePolicy = pexPolicy.Policy()
59 self.
persistence = Persistence.getPersistence(persistencePolicy)
62 return 'PosixStorage(root=%s)' % self.
root
65 def _pathFromURI(uri):
66 """Get the path part of the URI"""
67 return urllib.parse.urlparse(uri).path
71 """Get a relative path from a location to a location.
76 A path at which to start. It can be a relative path or an
79 A target location. It can be a relative path or an absolute path.
84 A relative path that describes the path from fromPath to toPath.
86 fromPath = os.path.realpath(fromPath)
87 return os.path.relpath(toPath, fromPath)
91 """Get an absolute path for the path from fromUri to toUri
95 fromPath : the starting location
96 A location at which to start. It can be a relative path or an
98 relativePath : the location relative to fromPath
104 Path that is an absolute path representation of fromPath +
105 relativePath, if one exists. If relativePath is absolute or if
106 fromPath is not related to relativePath then relativePath will be
109 if os.path.isabs(relativePath):
111 fromPath = os.path.realpath(fromPath)
112 return os.path.normpath(os.path.join(fromPath, relativePath))
115 def _getRepositoryCfg(uri):
116 """Get a persisted RepositoryCfg
120 uri : URI or path to a RepositoryCfg
125 A RepositoryCfg instance or None
128 parseRes = urllib.parse.urlparse(uri)
129 loc = os.path.join(parseRes.path,
'repositoryCfg.yaml')
130 if os.path.exists(loc):
131 with open(loc,
'r') as f:
132 repositoryCfg = yaml.load(f)
133 if repositoryCfg.root
is None:
134 repositoryCfg.root = uri
139 """Get a persisted RepositoryCfg
143 uri : URI or path to a RepositoryCfg
148 A RepositoryCfg instance or None
150 repositoryCfg = PosixStorage._getRepositoryCfg(uri)
151 if repositoryCfg
is not None:
158 """Serialize a RepositoryCfg to a location.
160 When loc == cfg.root, the RepositoryCfg is to be writtenat the root
161 location of the repository. In that case, root is not written, it is
162 implicit in the location of the cfg. This allows the cfg to move from
163 machine to machine without modification.
167 cfg : RepositoryCfg instance
168 The RepositoryCfg to be serailized.
170 The location to write the RepositoryCfg. If loc is None, the
171 location will be read from the root parameter of loc.
177 if loc
is None or cfg.root == loc:
183 parseRes = urllib.parse.urlparse(loc)
185 if not os.path.exists(loc):
187 loc = os.path.join(loc,
'repositoryCfg.yaml')
188 with safeFileIo.FileForWriteOnceCompareSame(loc)
as f:
193 """Get the mapper class associated with a repository root.
195 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
196 new code and repositories; they should use the Repository parentCfg mechanism.
201 The location of a persisted ReositoryCfg is (new style repos), or
202 the location where a _mapper file is (old style repos).
206 A class object or a class instance, depending on the state of the
207 mapper when the repository was created.
212 cfg = PosixStorage._getRepositoryCfg(root)
218 mapperFile =
"_mapper"
219 while not os.path.exists(os.path.join(basePath, mapperFile)):
221 if os.path.exists(os.path.join(basePath,
"_parent")):
222 basePath = os.path.join(basePath,
"_parent")
227 if mapperFile
is not None:
228 mapperFile = os.path.join(basePath, mapperFile)
231 with open(mapperFile,
"r") as f:
232 mapperName = f.readline().strip()
233 components = mapperName.split(".")
234 if len(components) <= 1:
235 raise RuntimeError(
"Unqualified mapper name %s in %s" %
236 (mapperName, mapperFile))
237 pkg = importlib.import_module(
".".join(components[:-1]))
238 return getattr(pkg, components[-1])
244 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
250 A path to the folder on the local filesystem.
255 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
258 linkpath = os.path.join(root,
'_parent')
259 if os.path.exists(linkpath):
261 return os.readlink(os.path.join(root,
'_parent'))
265 return os.path.join(root,
'_parent')
268 def write(self, butlerLocation, obj):
269 """Writes an object to a location and persistence format specified by
274 butlerLocation : ButlerLocation
275 The location & formatting for the object to be written.
276 obj : object instance
277 The object to be written.
279 self.log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
281 additionalData = butlerLocation.getAdditionalData()
282 storageName = butlerLocation.getStorageName()
283 locations = butlerLocation.getLocations()
285 pythonType = butlerLocation.getPythonType()
286 if pythonType
is not None:
287 if isinstance(pythonType, basestring):
289 pythonTypeTokenList = pythonType.split(
'.')
290 importClassString = pythonTypeTokenList.pop()
291 importClassString = importClassString.strip()
292 importPackage =
".".join(pythonTypeTokenList)
293 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
294 pythonType = getattr(importType, importClassString)
299 if hasattr(pythonType,
'butlerWrite'):
300 pythonType.butlerWrite(obj, butlerLocation=butlerLocation)
306 if storageName ==
"PickleStorage":
307 with open(logLoc.locString(),
"wb")
as outfile:
308 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
311 if storageName ==
"ConfigStorage":
312 obj.save(logLoc.locString())
315 if storageName ==
"FitsCatalogStorage":
316 flags = additionalData.getInt(
"flags", 0)
317 obj.writeFits(logLoc.locString(), flags=flags)
322 storage = self.persistence.getPersistStorage(storageName, logLoc)
323 storageList.append(storage)
325 if storageName ==
'FitsStorage':
326 self.persistence.persist(obj, storageList, additionalData)
330 if hasattr(obj,
'__deref__'):
332 self.persistence.persist(obj.__deref__(), storageList, additionalData)
334 self.persistence.persist(obj, storageList, additionalData)
336 def read(self, butlerLocation):
337 """Read from a butlerLocation.
341 butlerLocation : ButlerLocation
342 The location & formatting for the object(s) to be read.
346 A list of objects as described by the butler location. One item for
347 each location in butlerLocation.getLocations()
349 additionalData = butlerLocation.getAdditionalData()
351 storageName = butlerLocation.getStorageName()
353 locations = butlerLocation.getLocations()
354 pythonType = butlerLocation.getPythonType()
355 if pythonType
is not None:
356 if isinstance(pythonType, basestring):
358 pythonTypeTokenList = pythonType.split(
'.')
359 importClassString = pythonTypeTokenList.pop()
360 importClassString = importClassString.strip()
361 importPackage =
".".join(pythonTypeTokenList)
362 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
363 pythonType = getattr(importType, importClassString)
367 if hasattr(pythonType,
'butlerRead'):
368 results = pythonType.butlerRead(butlerLocation=butlerLocation)
371 for locationString
in locations:
372 locationString = os.path.join(self.
root, locationString)
376 if storageName ==
"PafStorage":
377 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
378 elif storageName ==
"YamlStorage":
379 finalItem = Policy(filePath=logLoc.locString())
380 elif storageName ==
"PickleStorage":
381 if not os.path.exists(logLoc.locString()):
382 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
383 with open(logLoc.locString(),
"rb")
as infile:
387 if sys.version_info.major >= 3:
388 finalItem = pickle.load(infile, encoding=
"latin1")
390 finalItem = pickle.load(infile)
391 elif storageName ==
"FitsCatalogStorage":
392 if not os.path.exists(logLoc.locString()):
393 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
395 hdu = additionalData.getInt(
"hdu", INT_MIN)
396 flags = additionalData.getInt(
"flags", 0)
397 finalItem = pythonType.readFits(logLoc.locString(), hdu, flags)
398 elif storageName ==
"ConfigStorage":
399 if not os.path.exists(logLoc.locString()):
400 raise RuntimeError(
"No such config file: " + logLoc.locString())
401 finalItem = pythonType()
402 finalItem.load(logLoc.locString())
405 storage = self.persistence.getRetrieveStorage(storageName, logLoc)
406 storageList.append(storage)
407 finalItem = self.persistence.unsafeRetrieve(
408 butlerLocation.getCppType(), storageList, additionalData)
409 results.append(finalItem)
414 """Implementaion of PosixStorage.exists for ButlerLocation objects."""
415 storageName = location.getStorageName()
416 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
417 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage'):
418 self.log.warn(
"butlerLocationExists for non-supported storage %s" % location)
420 for locationString
in location.getLocations():
421 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
428 """Check if location exists.
432 location : ButlerLocation or string
433 A a string or a ButlerLocation that describes the location of an
434 object in this storage.
439 True if exists, else False.
441 if isinstance(location, ButlerLocation):
448 """Get the full path to the location.
453 return os.path.join(self.
root, location)
457 """Test if a Version 1 Repository exists.
459 Version 1 Repositories only exist in posix storages and do not have a RepositoryCfg file.
460 To "exist" the folder at root must exist and contain files or folders.
465 A path to a folder on the local filesystem.
470 True if the repository at root exists, else False.
472 return os.path.exists(root)
and bool(os.listdir(root))
475 """Copy a file from one location to another on the local filesystem.
480 Path and name of existing file.
482 Path and name of new file.
488 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
491 """Get the path to a local copy of the file, downloading it to a
496 A path the the file in storage, relative to root.
500 A path to a local copy of the file. May be the original file (if
503 p = os.path.join(self.
root, path)
504 if os.path.exists(p):
510 """Search for the given path in this storage instance.
512 If the path contains an HDU indicator (a number in brackets before the
513 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
514 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
515 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
520 A filename (and optionally prefix path) to search for within root.
525 The location that was found, or None if no location was found.
530 def search(root, path, searchParents=False):
531 """Look for the given path in the current root.
533 Also supports searching for the path in Butler v1 repositories by
534 following the Butler v1 _parent symlink
536 If the path contains an HDU indicator (a number in brackets, e.g.
537 'foo.fits[1]', this will be stripped when searching and so
538 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
539 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
544 The path to the root directory.
546 The path to the file within the root directory.
547 searchParents : bool, optional
548 For Butler v1 repositories only, if true and a _parent symlink
549 exists, then the directory at _parent will be searched if the file
550 is not found in the root repository. Will continue searching the
551 parent of the parent until the file is found or no additional
557 The location that was found, or None if no location was found.
563 while len(rootDir) > 1
and rootDir[-1] ==
'/':
564 rootDir = rootDir[:-1]
566 if path.startswith(rootDir +
"/"):
568 path = path[len(rootDir +
'/'):]
570 elif rootDir ==
"/" and path.startswith(
"/"):
575 pathPrefix = os.path.dirname(path)
576 while pathPrefix !=
"" and pathPrefix !=
"/":
577 if os.path.realpath(pathPrefix) == os.path.realpath(root):
579 pathPrefix = os.path.dirname(pathPrefix)
580 if pathPrefix ==
"/":
582 elif pathPrefix !=
"":
583 path = path[len(pathPrefix)+1:]
589 firstBracket = path.find(
"[")
590 if firstBracket != -1:
591 strippedPath = path[:firstBracket]
592 pathStripped = path[firstBracket:]
596 paths = glob.glob(os.path.join(dir, strippedPath))
598 if pathPrefix != rootDir:
599 paths = [p[len(rootDir+
'/'):]
for p
in paths]
600 if pathStripped
is not None:
601 paths = [p + pathStripped
for p
in paths]
604 dir = os.path.join(dir,
"_parent")
605 if not os.path.exists(dir):
612 """Ask if a storage at the location described by uri exists
617 URI to the the root location of the storage
622 True if the storage exists, false if not
624 return os.path.exists(PosixStorage._pathFromURI(uri))
627 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
628 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
Class for logical location of a persisted Persistable instance.
Abstract base class for storage implementations.