24 from past.builtins
import basestring
33 from .
import (LogicalLocation, Persistence, Policy, StorageList,
34 StorageInterface, Storage, ButlerLocation,
35 NoRepositroyAtRoot, RepositoryCfg, doImport)
36 from lsst.log
import Log
37 import lsst.pex.policy
as pexPolicy
38 from .safeFileIo
import SafeFilename, safeMakeDir
41 __all__ = [
"PosixStorage"]
45 """Defines the interface for a storage location on the local filesystem.
50 URI or path that is used as the storage location.
52 If True a new repository will be created at the root location if it
53 does not exist. If False then a new repository will not be created.
58 If create is False and a repository does not exist at the root
59 specified by uri then NoRepositroyAtRoot is raised.
63 self.
log = Log.getLogger(
"daf.persistence.butler")
65 if self.
root and not os.path.exists(self.
root):
67 raise NoRepositroyAtRoot(
"No repository at {}".format(uri))
75 persistencePolicy = pexPolicy.Policy()
76 return Persistence.getPersistence(persistencePolicy)
79 return 'PosixStorage(root=%s)' % self.
root
82 def _pathFromURI(uri):
83 """Get the path part of the URI"""
84 return urllib.parse.urlparse(uri).path
88 """Get a relative path from a location to a location.
93 A path at which to start. It can be a relative path or an
96 A target location. It can be a relative path or an absolute path.
101 A relative path that describes the path from fromPath to toPath.
103 fromPath = os.path.realpath(fromPath)
104 return os.path.relpath(toPath, fromPath)
108 """Get an absolute path for the path from fromUri to toUri
112 fromPath : the starting location
113 A location at which to start. It can be a relative path or an
115 relativePath : the location relative to fromPath
121 Path that is an absolute path representation of fromPath +
122 relativePath, if one exists. If relativePath is absolute or if
123 fromPath is not related to relativePath then relativePath will be
126 if os.path.isabs(relativePath):
128 fromPath = os.path.realpath(fromPath)
129 return os.path.normpath(os.path.join(fromPath, relativePath))
133 """Get a persisted RepositoryCfg
137 uri : URI or path to a RepositoryCfg
142 A RepositoryCfg instance or None
144 storage = Storage.makeFromURI(uri)
145 location = ButlerLocation(pythonType=RepositoryCfg,
148 locationList=
'repositoryCfg.yaml',
154 return storage.read(location)
158 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
159 location = ButlerLocation(pythonType=RepositoryCfg,
162 locationList=
'repositoryCfg.yaml',
168 storage.write(location, cfg)
172 """Get the mapper class associated with a repository root.
174 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
175 new code and repositories; they should use the Repository parentCfg mechanism.
180 The location of a persisted ReositoryCfg is (new style repos), or
181 the location where a _mapper file is (old style repos).
185 A class object or a class instance, depending on the state of the
186 mapper when the repository was created.
191 cfg = PosixStorage.getRepositoryCfg(root)
197 mapperFile =
"_mapper"
198 while not os.path.exists(os.path.join(basePath, mapperFile)):
200 if os.path.exists(os.path.join(basePath,
"_parent")):
201 basePath = os.path.join(basePath,
"_parent")
206 if mapperFile
is not None:
207 mapperFile = os.path.join(basePath, mapperFile)
210 with open(mapperFile,
"r") as f:
211 mapperName = f.readline().strip()
212 components = mapperName.split(".")
213 if len(components) <= 1:
214 raise RuntimeError(
"Unqualified mapper name %s in %s" %
215 (mapperName, mapperFile))
216 pkg = importlib.import_module(
".".join(components[:-1]))
217 return getattr(pkg, components[-1])
223 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
229 A path to the folder on the local filesystem.
234 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
237 linkpath = os.path.join(root,
'_parent')
238 if os.path.exists(linkpath):
240 return os.readlink(os.path.join(root,
'_parent'))
244 return os.path.join(root,
'_parent')
247 def write(self, butlerLocation, obj):
248 """Writes an object to a location and persistence format specified by
253 butlerLocation : ButlerLocation
254 The location & formatting for the object to be written.
255 obj : object instance
256 The object to be written.
258 self.log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
260 writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
261 if not writeFormatter:
262 writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
264 writeFormatter(butlerLocation, obj)
267 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
269 def read(self, butlerLocation):
270 """Read from a butlerLocation.
274 butlerLocation : ButlerLocation
275 The location & formatting for the object(s) to be read.
279 A list of objects as described by the butler location. One item for
280 each location in butlerLocation.getLocations()
282 readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
283 if not readFormatter:
284 readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
286 return readFormatter(butlerLocation)
288 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
291 """Implementaion of PosixStorage.exists for ButlerLocation objects."""
292 storageName = location.getStorageName()
293 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
294 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage'):
295 self.log.warn(
"butlerLocationExists for non-supported storage %s" % location)
297 for locationString
in location.getLocations():
298 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
305 """Check if location exists.
309 location : ButlerLocation or string
310 A a string or a ButlerLocation that describes the location of an
311 object in this storage.
316 True if exists, else False.
318 if isinstance(location, ButlerLocation):
325 """Get the full path to the location.
330 return os.path.join(self.
root, location)
334 """Test if a Version 1 Repository exists.
336 Version 1 Repositories only exist in posix storages, do not have a
337 RepositoryCfg file, and contain either a registry.sqlite3 file, a
338 _mapper file, or a _parent link.
343 A path to a folder on the local filesystem.
348 True if the repository at root exists, else False.
350 return os.path.exists(root)
and (
351 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or
352 os.path.exists(os.path.join(root,
"_mapper"))
or
353 os.path.exists(os.path.join(root,
"_parent"))
357 """Copy a file from one location to another on the local filesystem.
362 Path and name of existing file.
364 Path and name of new file.
370 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
373 """Get a handle to a local copy of the file, downloading it to a
378 A path the the file in storage, relative to root.
382 A handle to a local copy of the file. If storage is remote it will be
383 a temporary file. If storage is local it may be the original file or
384 a temporary file. The file name can be gotten via the 'name' property
385 of the returned object.
387 p = os.path.join(self.
root, path)
397 """Search for the given path in this storage instance.
399 If the path contains an HDU indicator (a number in brackets before the
400 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
401 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
402 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
407 A filename (and optionally prefix path) to search for within root.
412 The location that was found, or None if no location was found.
417 def search(root, path, searchParents=False):
418 """Look for the given path in the current root.
420 Also supports searching for the path in Butler v1 repositories by
421 following the Butler v1 _parent symlink
423 If the path contains an HDU indicator (a number in brackets, e.g.
424 'foo.fits[1]', this will be stripped when searching and so
425 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
426 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
431 The path to the root directory.
433 The path to the file within the root directory.
434 searchParents : bool, optional
435 For Butler v1 repositories only, if true and a _parent symlink
436 exists, then the directory at _parent will be searched if the file
437 is not found in the root repository. Will continue searching the
438 parent of the parent until the file is found or no additional
444 The location that was found, or None if no location was found.
450 while len(rootDir) > 1
and rootDir[-1] ==
'/':
451 rootDir = rootDir[:-1]
453 if path.startswith(rootDir +
"/"):
455 path = path[len(rootDir +
'/'):]
457 elif rootDir ==
"/" and path.startswith(
"/"):
462 pathPrefix = os.path.dirname(path)
463 while pathPrefix !=
"" and pathPrefix !=
"/":
464 if os.path.realpath(pathPrefix) == os.path.realpath(root):
466 pathPrefix = os.path.dirname(pathPrefix)
467 if pathPrefix ==
"/":
469 elif pathPrefix !=
"":
470 path = path[len(pathPrefix)+1:]
476 firstBracket = path.find(
"[")
477 if firstBracket != -1:
478 strippedPath = path[:firstBracket]
479 pathStripped = path[firstBracket:]
483 paths = glob.glob(os.path.join(dir, strippedPath))
485 if pathPrefix != rootDir:
486 paths = [p[len(rootDir+
'/'):]
for p
in paths]
487 if pathStripped
is not None:
488 paths = [p + pathStripped
for p
in paths]
491 dir = os.path.join(dir,
"_parent")
492 if not os.path.exists(dir):
499 """Ask if a storage at the location described by uri exists
504 URI to the the root location of the storage
509 True if the storage exists, false if not
511 return os.path.exists(PosixStorage._pathFromURI(uri))
515 """Read from a butlerLocation.
519 butlerLocation : ButlerLocation
520 The location & formatting for the object(s) to be read.
524 A list of objects as described by the butler location. One item for
525 each location in butlerLocation.getLocations()
528 for locationString
in butlerLocation.getLocations():
529 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
530 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
531 if not os.path.exists(logLoc.locString()):
532 raise RuntimeError(
"No such config file: " + logLoc.locString())
533 pythonType = butlerLocation.getPythonType()
534 if pythonType
is not None:
535 if isinstance(pythonType, basestring):
537 finalItem = pythonType()
538 finalItem.load(logLoc.locString())
539 results.append(finalItem)
544 """Writes an object to a location and persistence format specified by
549 butlerLocation : ButlerLocation
550 The location & formatting for the object to be written.
551 obj : object instance
552 The object to be written.
554 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
556 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
557 obj.save(logLoc.locString())
561 """Read from a butlerLocation.
565 butlerLocation : ButlerLocation
566 The location & formatting for the object(s) to be read.
570 A list of objects as described by the butler location. One item for
571 each location in butlerLocation.getLocations()
574 for locationString
in butlerLocation.getLocations():
575 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
576 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
578 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
580 storageList.append(storage)
581 finalItem = PosixStorage.getPersistence().unsafeRetrieve(
582 butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
583 results.append(finalItem)
588 """Writes an object to a location and persistence format specified by
593 butlerLocation : ButlerLocation
594 The location & formatting for the object to be written.
595 obj : object instance
596 The object to be written.
598 location = butlerLocation.getLocations()[0]
599 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, location))
as locationString:
600 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
603 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
604 storageList.append(storage)
605 persistence = PosixStorage.getPersistence()
606 if hasattr(obj,
'__deref__'):
608 persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
610 persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
614 """Read from a butlerLocation.
618 butlerLocation : ButlerLocation
619 The location & formatting for the object(s) to be read.
623 A list of objects as described by the butler location. One item for
624 each location in butlerLocation.getLocations()
628 additionalData = butlerLocation.getAdditionalData()
629 for locationString
in butlerLocation.getLocations():
630 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
632 if not os.path.exists(logLoc.locString()):
633 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
634 with open(logLoc.locString(),
"rb")
as infile:
638 if sys.version_info.major >= 3:
639 finalItem = pickle.load(infile, encoding=
"latin1")
641 finalItem = pickle.load(infile)
642 results.append(finalItem)
647 """Writes an object to a location and persistence format specified by
652 butlerLocation : ButlerLocation
653 The location & formatting for the object to be written.
654 obj : object instance
655 The object to be written.
657 additionalData = butlerLocation.getAdditionalData()
658 locations = butlerLocation.getLocations()
659 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
661 with open(logLoc.locString(),
"wb")
as outfile:
662 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
666 """Read from a butlerLocation.
670 butlerLocation : ButlerLocation
671 The location & formatting for the object(s) to be read.
675 A list of objects as described by the butler location. One item for
676 each location in butlerLocation.getLocations()
678 pythonType = butlerLocation.getPythonType()
679 if pythonType
is not None:
680 if isinstance(pythonType, basestring):
683 additionalData = butlerLocation.getAdditionalData()
684 for locationString
in butlerLocation.getLocations():
685 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
687 if not os.path.exists(logLoc.locString()):
688 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
690 if additionalData.exists(
"hdu"):
691 kwds[
"hdu"] = additionalData.getInt(
"hdu")
692 if additionalData.exists(
"flags"):
693 kwds[
"flags"] = additionalData.getInt(
"flags")
694 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
695 results.append(finalItem)
700 """Writes an object to a location and persistence format specified by
705 butlerLocation : ButlerLocation
706 The location & formatting for the object to be written.
707 obj : object instance
708 The object to be written.
710 additionalData = butlerLocation.getAdditionalData()
711 locations = butlerLocation.getLocations()
712 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
714 if additionalData.exists(
"flags"):
715 kwds = dict(flags=additionalData.getInt(
"flags"))
718 obj.writeFits(logLoc.locString(), **kwds)
723 """Read from a butlerLocation.
727 butlerLocation : ButlerLocation
728 The location & formatting for the object(s) to be read.
732 A list of objects as described by the butler location. One item for
733 each location in butlerLocation.getLocations()
736 for locationString
in butlerLocation.getLocations():
737 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
738 butlerLocation.getAdditionalData())
739 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
740 results.append(finalItem)
745 """Read from a butlerLocation.
749 butlerLocation : ButlerLocation
750 The location & formatting for the object(s) to be read.
754 A list of objects as described by the butler location. One item for
755 each location in butlerLocation.getLocations()
758 for locationString
in butlerLocation.getLocations():
759 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
760 butlerLocation.getAdditionalData())
761 finalItem = Policy(filePath=logLoc.locString())
762 results.append(finalItem)
768 additionalData = butlerLocation.getAdditionalData()
769 for locationString
in butlerLocation.getLocations():
770 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
771 butlerLocation.getAdditionalData())
773 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
774 storageList.append(storage)
775 finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
777 results.append(finalItem)
782 additionalData = butlerLocation.getAdditionalData()
783 location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
788 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
789 storageList.append(storage)
791 if hasattr(obj,
'__deref__'):
793 PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
795 PosixStorage.getPersistence().persist(obj, storageList, additionalData)
798 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
799 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
800 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
801 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
802 PosixStorage.registerFormatters(
"PafStorage", writeFormatter=readPafStorage)
803 PosixStorage.registerFormatters(
"YamlStorage", readFormatter=readYamlStorage)
804 PosixStorage.registerFormatters(
"BoostStorage", readFitsStorage, writeFitsStorage)
806 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
807 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
Class for logical location of a persisted Persistable instance.
def writeFitsCatalogStorage
def readFitsCatalogStorage