24 from past.builtins
import basestring
34 from .
import (LogicalLocation, Persistence, Policy, StorageList,
35 StorageInterface, Storage, ButlerLocation,
36 NoRepositroyAtRoot, RepositoryCfg, doImport)
39 from .safeFileIo
import SafeFilename, safeMakeDir
42 __all__ = [
"PosixStorage"]
46 """Defines the interface for a storage location on the local filesystem. 51 URI or path that is used as the storage location. 53 If True a new repository will be created at the root location if it 54 does not exist. If False then a new repository will not be created. 59 If create is False and a repository does not exist at the root 60 specified by uri then NoRepositroyAtRoot is raised. 64 self.
log = Log.getLogger(
"daf.persistence.butler")
66 if self.
root and not os.path.exists(self.
root):
77 return Persistence.getPersistence(persistencePolicy)
80 return 'PosixStorage(root=%s)' % self.
root 83 def _pathFromURI(uri):
84 """Get the path part of the URI""" 85 return urllib.parse.urlparse(uri).path
89 """Get a relative path from a location to a location. 94 A path at which to start. It can be a relative path or an 97 A target location. It can be a relative path or an absolute path. 102 A relative path that describes the path from fromPath to toPath. 104 fromPath = os.path.realpath(fromPath)
105 return os.path.relpath(toPath, fromPath)
109 """Get an absolute path for the path from fromUri to toUri 113 fromPath : the starting location 114 A location at which to start. It can be a relative path or an 116 relativePath : the location relative to fromPath 122 Path that is an absolute path representation of fromPath + 123 relativePath, if one exists. If relativePath is absolute or if 124 fromPath is not related to relativePath then relativePath will be 127 if os.path.isabs(relativePath):
129 fromPath = os.path.realpath(fromPath)
130 return os.path.normpath(os.path.join(fromPath, relativePath))
134 """Get a persisted RepositoryCfg 138 uri : URI or path to a RepositoryCfg 143 A RepositoryCfg instance or None 145 storage = Storage.makeFromURI(uri)
149 locationList=
'repositoryCfg.yaml',
155 return storage.read(location)
159 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
163 locationList=
'repositoryCfg.yaml',
169 storage.write(location, cfg)
173 """Get the mapper class associated with a repository root. 175 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 176 new code and repositories; they should use the Repository parentCfg mechanism. 181 The location of a persisted ReositoryCfg is (new style repos), or 182 the location where a _mapper file is (old style repos). 186 A class object or a class instance, depending on the state of the 187 mapper when the repository was created. 192 cfg = PosixStorage.getRepositoryCfg(root)
198 mapperFile =
"_mapper" 199 while not os.path.exists(os.path.join(basePath, mapperFile)):
201 if os.path.exists(os.path.join(basePath,
"_parent")):
202 basePath = os.path.join(basePath,
"_parent")
207 if mapperFile
is not None:
208 mapperFile = os.path.join(basePath, mapperFile)
211 with open(mapperFile,
"r") as f: 212 mapperName = f.readline().strip() 213 components = mapperName.split(".")
214 if len(components) <= 1:
215 raise RuntimeError(
"Unqualified mapper name %s in %s" %
216 (mapperName, mapperFile))
217 pkg = importlib.import_module(
".".join(components[:-1]))
218 return getattr(pkg, components[-1])
224 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 230 A path to the folder on the local filesystem. 235 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 238 linkpath = os.path.join(root,
'_parent')
239 if os.path.exists(linkpath):
241 return os.readlink(os.path.join(root,
'_parent'))
245 return os.path.join(root,
'_parent')
248 def write(self, butlerLocation, obj):
249 """Writes an object to a location and persistence format specified by 254 butlerLocation : ButlerLocation 255 The location & formatting for the object to be written. 256 obj : object instance 257 The object to be written. 259 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
262 if not writeFormatter:
265 writeFormatter(butlerLocation, obj)
268 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
270 def read(self, butlerLocation):
271 """Read from a butlerLocation. 275 butlerLocation : ButlerLocation 276 The location & formatting for the object(s) to be read. 280 A list of objects as described by the butler location. One item for 281 each location in butlerLocation.getLocations() 284 if not readFormatter:
287 return readFormatter(butlerLocation)
289 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
292 """Implementation of PosixStorage.exists for ButlerLocation objects. 294 storageName = location.getStorageName()
295 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
296 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
297 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
298 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
300 for locationString
in location.getLocations():
301 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
308 """Check if location exists. 312 location : ButlerLocation or string 313 A a string or a ButlerLocation that describes the location of an 314 object in this storage. 319 True if exists, else False. 321 if isinstance(location, ButlerLocation):
328 """Get the full path to the location. 333 return os.path.join(self.
root, location)
337 """Test if a Version 1 Repository exists. 339 Version 1 Repositories only exist in posix storages, do not have a 340 RepositoryCfg file, and contain either a registry.sqlite3 file, a 341 _mapper file, or a _parent link. 346 A path to a folder on the local filesystem. 351 True if the repository at root exists, else False. 353 return os.path.exists(root)
and (
354 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or 355 os.path.exists(os.path.join(root,
"_mapper"))
or 356 os.path.exists(os.path.join(root,
"_parent"))
360 """Copy a file from one location to another on the local filesystem. 365 Path and name of existing file. 367 Path and name of new file. 373 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
376 """Get a handle to a local copy of the file, downloading it to a 381 A path the the file in storage, relative to root. 385 A handle to a local copy of the file. If storage is remote it will be 386 a temporary file. If storage is local it may be the original file or 387 a temporary file. The file name can be gotten via the 'name' property 388 of the returned object. 390 p = os.path.join(self.
root, path)
400 """Search for the given path in this storage instance. 402 If the path contains an HDU indicator (a number in brackets before the 403 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 404 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 405 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 410 A filename (and optionally prefix path) to search for within root. 415 The location that was found, or None if no location was found. 420 def search(root, path, searchParents=False):
421 """Look for the given path in the current root. 423 Also supports searching for the path in Butler v1 repositories by 424 following the Butler v1 _parent symlink 426 If the path contains an HDU indicator (a number in brackets, e.g. 427 'foo.fits[1]', this will be stripped when searching and so 428 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 429 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 434 The path to the root directory. 436 The path to the file within the root directory. 437 searchParents : bool, optional 438 For Butler v1 repositories only, if true and a _parent symlink 439 exists, then the directory at _parent will be searched if the file 440 is not found in the root repository. Will continue searching the 441 parent of the parent until the file is found or no additional 447 The location that was found, or None if no location was found. 453 while len(rootDir) > 1
and rootDir[-1] ==
'/':
454 rootDir = rootDir[:-1]
456 if path.startswith(rootDir +
"/"):
458 path = path[len(rootDir +
'/'):]
460 elif rootDir ==
"/" and path.startswith(
"/"):
465 pathPrefix = os.path.dirname(path)
466 while pathPrefix !=
"" and pathPrefix !=
"/":
467 if os.path.realpath(pathPrefix) == os.path.realpath(root):
469 pathPrefix = os.path.dirname(pathPrefix)
470 if pathPrefix ==
"/":
472 elif pathPrefix !=
"":
473 path = path[len(pathPrefix)+1:]
479 firstBracket = path.find(
"[")
480 if firstBracket != -1:
481 strippedPath = path[:firstBracket]
482 pathStripped = path[firstBracket:]
486 paths = glob.glob(os.path.join(dir, strippedPath))
488 if pathPrefix != rootDir:
489 paths = [p[len(rootDir+
'/'):]
for p
in paths]
490 if pathStripped
is not None:
491 paths = [p + pathStripped
for p
in paths]
494 dir = os.path.join(dir,
"_parent")
495 if not os.path.exists(dir):
502 """Ask if a storage at the location described by uri exists 507 URI to the the root location of the storage 512 True if the storage exists, false if not 514 return os.path.exists(PosixStorage._pathFromURI(uri))
518 """Read an lsst.pex.config.Config from a butlerLocation. 522 butlerLocation : ButlerLocation 523 The location for the object(s) to be read. 527 A list of objects as described by the butler location. One item for 528 each location in butlerLocation.getLocations() 531 for locationString
in butlerLocation.getLocations():
532 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
533 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
534 if not os.path.exists(logLoc.locString()):
535 raise RuntimeError(
"No such config file: " + logLoc.locString())
536 pythonType = butlerLocation.getPythonType()
537 if pythonType
is not None:
538 if isinstance(pythonType, basestring):
540 finalItem = pythonType()
541 finalItem.load(logLoc.locString())
542 results.append(finalItem)
547 """Writes an lsst.pex.config.Config object to a location specified by 552 butlerLocation : ButlerLocation 553 The location for the object to be written. 554 obj : object instance 555 The object to be written. 557 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
559 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
560 obj.save(logLoc.locString())
564 """Read a FITS image from a butlerLocation. 568 butlerLocation : ButlerLocation 569 The location for the object(s) to be read. 573 A list of objects as described by the butler location. One item for 574 each location in butlerLocation.getLocations() 577 for locationString
in butlerLocation.getLocations():
578 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
579 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
581 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
583 storageList.append(storage)
584 finalItem = PosixStorage.getPersistence().unsafeRetrieve(
585 butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
586 results.append(finalItem)
591 """Writes an object to a FITS file specified by ButlerLocation. 595 butlerLocation : ButlerLocation 596 The location for the object to be written. 597 obj : object instance 598 The object to be written. 600 location = butlerLocation.getLocations()[0]
601 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, location))
as locationString:
602 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
605 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
606 storageList.append(storage)
607 persistence = PosixStorage.getPersistence()
608 if hasattr(obj,
'__deref__'):
610 persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
612 persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
616 """Read a catalog from a Parquet file specified by ButlerLocation. 618 The object returned by this is expected to be a subtype 619 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` 620 that allows for lazy loading of the data. 624 butlerLocation : ButlerLocation 625 The location for the object(s) to be read. 629 A list of objects as described by the butler location. One item for 630 each location in butlerLocation.getLocations() 633 additionalData = butlerLocation.getAdditionalData()
635 for locationString
in butlerLocation.getLocations():
636 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
638 if not os.path.exists(logLoc.locString()):
639 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
641 pythonType = butlerLocation.getPythonType()
642 if pythonType
is not None:
643 if isinstance(pythonType, basestring):
646 filename = logLoc.locString()
650 results.append(pythonType(filename=filename))
656 """Writes pandas dataframe to parquet file. 660 butlerLocation : ButlerLocation 661 The location for the object(s) to be read. 662 obj : `lsst.qa.explorer.parquetTable.ParquetTable` 663 Wrapped DataFrame to write. 666 additionalData = butlerLocation.getAdditionalData()
667 locations = butlerLocation.getLocations()
668 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
670 filename = logLoc.locString()
675 """Writes an object to a YAML file specified by ButlerLocation. 679 butlerLocation : ButlerLocation 680 The location for the object to be written. 681 obj : object instance 682 The object to be written. 684 additionalData = butlerLocation.getAdditionalData()
685 locations = butlerLocation.getLocations()
686 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
688 with open(logLoc.locString(),
"w")
as outfile:
689 yaml.dump(obj, outfile)
693 """Read an object from a pickle file specified by ButlerLocation. 697 butlerLocation : ButlerLocation 698 The location for the object(s) to be read. 702 A list of objects as described by the butler location. One item for 703 each location in butlerLocation.getLocations() 707 additionalData = butlerLocation.getAdditionalData()
708 for locationString
in butlerLocation.getLocations():
709 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
711 if not os.path.exists(logLoc.locString()):
712 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
713 with open(logLoc.locString(),
"rb")
as infile:
717 if sys.version_info.major >= 3:
718 finalItem = pickle.load(infile, encoding=
"latin1")
720 finalItem = pickle.load(infile)
721 results.append(finalItem)
726 """Writes an object to a pickle file specified by ButlerLocation. 730 butlerLocation : ButlerLocation 731 The location for the object to be written. 732 obj : object instance 733 The object to be written. 735 additionalData = butlerLocation.getAdditionalData()
736 locations = butlerLocation.getLocations()
737 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
739 with open(logLoc.locString(),
"wb")
as outfile:
740 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
744 """Read a catalog from a FITS table specified by ButlerLocation. 748 butlerLocation : ButlerLocation 749 The location for the object(s) to be read. 753 A list of objects as described by the butler location. One item for 754 each location in butlerLocation.getLocations() 756 pythonType = butlerLocation.getPythonType()
757 if pythonType
is not None:
758 if isinstance(pythonType, basestring):
761 additionalData = butlerLocation.getAdditionalData()
762 for locationString
in butlerLocation.getLocations():
763 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
765 if not os.path.exists(logLoc.locString()):
766 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
768 if additionalData.exists(
"hdu"):
769 kwds[
"hdu"] = additionalData.getInt(
"hdu")
770 if additionalData.exists(
"flags"):
771 kwds[
"flags"] = additionalData.getInt(
"flags")
772 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
773 results.append(finalItem)
778 """Writes a catalog to a FITS table specified by ButlerLocation. 782 butlerLocation : ButlerLocation 783 The location for the object to be written. 784 obj : object instance 785 The object to be written. 787 additionalData = butlerLocation.getAdditionalData()
788 locations = butlerLocation.getLocations()
789 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
791 if additionalData.exists(
"flags"):
792 kwds = dict(flags=additionalData.getInt(
"flags"))
795 obj.writeFits(logLoc.locString(), **kwds)
799 """Read from a butlerLocation (always fails for this storage type). 803 butlerLocation : ButlerLocation 804 The location for the object(s) to be read. 808 A list of objects as described by the butler location. One item for 809 each location in butlerLocation.getLocations() 811 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
815 """Writes a matplotlib.figure.Figure to a location, using the template's 816 filename suffix to infer the file format. 820 butlerLocation : ButlerLocation 821 The location for the object to be written. 822 obj : matplotlib.figure.Figure 823 The object to be written. 825 additionalData = butlerLocation.getAdditionalData()
826 locations = butlerLocation.getLocations()
827 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
833 _, ext = os.path.splitext(locations[0])
840 obj.savefig(logLoc.locString(), format=ext)
844 """Read a policy from a PAF file specified by a ButlerLocation. 848 butlerLocation : ButlerLocation 849 The location for the object(s) to be read. 853 A list of objects as described by the butler location. One item for 854 each location in butlerLocation.getLocations() 857 for locationString
in butlerLocation.getLocations():
858 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
859 butlerLocation.getAdditionalData())
860 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
861 results.append(finalItem)
866 """Read an object from a YAML file specified by a butlerLocation. 870 butlerLocation : ButlerLocation 871 The location for the object(s) to be read. 875 A list of objects as described by the butler location. One item for 876 each location in butlerLocation.getLocations() 879 for locationString
in butlerLocation.getLocations():
880 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
881 butlerLocation.getAdditionalData())
882 if not os.path.exists(logLoc.locString()):
883 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
885 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
886 finalItem =
Policy(filePath=logLoc.locString())
888 with open(logLoc.locString(),
"rb")
as infile:
889 finalItem = yaml.load(infile)
890 results.append(finalItem)
895 """Read an object from a boost::serialization file. 899 butlerLocation : ButlerLocation 900 The location for the object(s) to be read. 904 A list of objects as described by the butler location. One item for 905 each location in butlerLocation.getLocations() 908 additionalData = butlerLocation.getAdditionalData()
909 for locationString
in butlerLocation.getLocations():
910 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
911 butlerLocation.getAdditionalData())
913 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
914 storageList.append(storage)
915 finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
917 results.append(finalItem)
922 """Writes an object via boost::serialization. 926 butlerLocation : ButlerLocation 927 The location for the object to be written. 928 obj : object instance 929 The object to be written. 931 additionalData = butlerLocation.getAdditionalData()
932 location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
937 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
938 storageList.append(storage)
940 if hasattr(obj,
'__deref__'):
942 PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
944 PosixStorage.getPersistence().persist(obj, storageList, additionalData)
947 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
948 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
949 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
950 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
951 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
952 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
953 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
954 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
955 PosixStorage.registerFormatters(
"BoostStorage", readFitsStorage, writeFitsStorage)
957 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
958 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def safeMakeDir(directory)
def writeMatplotlibStorage(butlerLocation, obj)
def getWriteFormatter(cls, objType)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def relativePath(fromPath, toPath)
def writeFitsCatalogStorage(butlerLocation, obj)
def exists(self, location)
def readPafStorage(butlerLocation)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def locationWithRoot(self, location)
def readFitsCatalogStorage(butlerLocation)
def getParentSymlinkPath(root)
def readYamlStorage(butlerLocation)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getReadFormatter(cls, objType)
def writeYamlStorage(butlerLocation, obj)
def getLocalFile(self, path)
def read(self, butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)