24 from past.builtins
import basestring
35 from .
import (LogicalLocation, Persistence, Policy, StorageList,
36 StorageInterface, Storage, ButlerLocation,
37 NoRepositroyAtRoot, RepositoryCfg, doImport)
40 from .safeFileIo
import SafeFilename, safeMakeDir
43 __all__ = [
"PosixStorage"]
47 """Defines the interface for a storage location on the local filesystem. 52 URI or path that is used as the storage location. 54 If True a new repository will be created at the root location if it 55 does not exist. If False then a new repository will not be created. 60 If create is False and a repository does not exist at the root 61 specified by uri then NoRepositroyAtRoot is raised. 65 self.
log = Log.getLogger(
"daf.persistence.butler")
67 if self.
root and not os.path.exists(self.
root):
78 return Persistence.getPersistence(persistencePolicy)
81 return 'PosixStorage(root=%s)' % self.
root 84 def _pathFromURI(uri):
85 """Get the path part of the URI""" 86 return urllib.parse.urlparse(uri).path
90 """Get a relative path from a location to a location. 95 A path at which to start. It can be a relative path or an 98 A target location. It can be a relative path or an absolute path. 103 A relative path that describes the path from fromPath to toPath. 105 fromPath = os.path.realpath(fromPath)
106 return os.path.relpath(toPath, fromPath)
110 """Get an absolute path for the path from fromUri to toUri 114 fromPath : the starting location 115 A location at which to start. It can be a relative path or an 117 relativePath : the location relative to fromPath 123 Path that is an absolute path representation of fromPath + 124 relativePath, if one exists. If relativePath is absolute or if 125 fromPath is not related to relativePath then relativePath will be 128 if os.path.isabs(relativePath):
130 fromPath = os.path.realpath(fromPath)
131 return os.path.normpath(os.path.join(fromPath, relativePath))
135 """Get a persisted RepositoryCfg 139 uri : URI or path to a RepositoryCfg 144 A RepositoryCfg instance or None 146 storage = Storage.makeFromURI(uri)
150 locationList=
'repositoryCfg.yaml',
156 return storage.read(location)
160 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
164 locationList=
'repositoryCfg.yaml',
170 storage.write(location, cfg)
174 """Get the mapper class associated with a repository root. 176 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 177 new code and repositories; they should use the Repository parentCfg mechanism. 182 The location of a persisted ReositoryCfg is (new style repos), or 183 the location where a _mapper file is (old style repos). 187 A class object or a class instance, depending on the state of the 188 mapper when the repository was created. 193 cfg = PosixStorage.getRepositoryCfg(root)
199 mapperFile =
"_mapper" 200 while not os.path.exists(os.path.join(basePath, mapperFile)):
202 if os.path.exists(os.path.join(basePath,
"_parent")):
203 basePath = os.path.join(basePath,
"_parent")
208 if mapperFile
is not None:
209 mapperFile = os.path.join(basePath, mapperFile)
212 with open(mapperFile,
"r") as f: 213 mapperName = f.readline().strip() 214 components = mapperName.split(".")
215 if len(components) <= 1:
216 raise RuntimeError(
"Unqualified mapper name %s in %s" %
217 (mapperName, mapperFile))
218 pkg = importlib.import_module(
".".join(components[:-1]))
219 return getattr(pkg, components[-1])
225 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 231 A path to the folder on the local filesystem. 236 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 239 linkpath = os.path.join(root,
'_parent')
240 if os.path.exists(linkpath):
242 return os.readlink(os.path.join(root,
'_parent'))
246 return os.path.join(root,
'_parent')
249 def write(self, butlerLocation, obj):
250 """Writes an object to a location and persistence format specified by 255 butlerLocation : ButlerLocation 256 The location & formatting for the object to be written. 257 obj : object instance 258 The object to be written. 260 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
263 if not writeFormatter:
266 writeFormatter(butlerLocation, obj)
269 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
271 def read(self, butlerLocation):
272 """Read from a butlerLocation. 276 butlerLocation : ButlerLocation 277 The location & formatting for the object(s) to be read. 281 A list of objects as described by the butler location. One item for 282 each location in butlerLocation.getLocations() 285 if not readFormatter:
288 return readFormatter(butlerLocation)
290 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
293 """Implementation of PosixStorage.exists for ButlerLocation objects. 295 storageName = location.getStorageName()
296 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
297 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
298 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
299 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
301 for locationString
in location.getLocations():
302 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
309 """Check if location exists. 313 location : ButlerLocation or string 314 A a string or a ButlerLocation that describes the location of an 315 object in this storage. 320 True if exists, else False. 322 if isinstance(location, ButlerLocation):
329 """Get the full path to the location. 334 return os.path.join(self.
root, location)
338 """Test if a Version 1 Repository exists. 340 Version 1 Repositories only exist in posix storages, do not have a 341 RepositoryCfg file, and contain either a registry.sqlite3 file, a 342 _mapper file, or a _parent link. 347 A path to a folder on the local filesystem. 352 True if the repository at root exists, else False. 354 return os.path.exists(root)
and (
355 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or 356 os.path.exists(os.path.join(root,
"_mapper"))
or 357 os.path.exists(os.path.join(root,
"_parent"))
361 """Copy a file from one location to another on the local filesystem. 366 Path and name of existing file. 368 Path and name of new file. 374 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
377 """Get a handle to a local copy of the file, downloading it to a 382 A path the the file in storage, relative to root. 386 A handle to a local copy of the file. If storage is remote it will be 387 a temporary file. If storage is local it may be the original file or 388 a temporary file. The file name can be gotten via the 'name' property 389 of the returned object. 391 p = os.path.join(self.
root, path)
401 """Search for the given path in this storage instance. 403 If the path contains an HDU indicator (a number in brackets before the 404 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 405 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 406 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 411 A filename (and optionally prefix path) to search for within root. 416 The location that was found, or None if no location was found. 421 def search(root, path, searchParents=False):
422 """Look for the given path in the current root. 424 Also supports searching for the path in Butler v1 repositories by 425 following the Butler v1 _parent symlink 427 If the path contains an HDU indicator (a number in brackets, e.g. 428 'foo.fits[1]', this will be stripped when searching and so 429 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 430 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 435 The path to the root directory. 437 The path to the file within the root directory. 438 searchParents : bool, optional 439 For Butler v1 repositories only, if true and a _parent symlink 440 exists, then the directory at _parent will be searched if the file 441 is not found in the root repository. Will continue searching the 442 parent of the parent until the file is found or no additional 448 The location that was found, or None if no location was found. 454 while len(rootDir) > 1
and rootDir[-1] ==
'/':
455 rootDir = rootDir[:-1]
457 if path.startswith(rootDir +
"/"):
459 path = path[len(rootDir +
'/'):]
461 elif rootDir ==
"/" and path.startswith(
"/"):
466 pathPrefix = os.path.dirname(path)
467 while pathPrefix !=
"" and pathPrefix !=
"/":
468 if os.path.realpath(pathPrefix) == os.path.realpath(root):
470 pathPrefix = os.path.dirname(pathPrefix)
471 if pathPrefix ==
"/":
473 elif pathPrefix !=
"":
474 path = path[len(pathPrefix)+1:]
480 firstBracket = path.find(
"[")
481 if firstBracket != -1:
482 strippedPath = path[:firstBracket]
483 pathStripped = path[firstBracket:]
487 paths = glob.glob(os.path.join(dir, strippedPath))
489 if pathPrefix != rootDir:
490 paths = [p[len(rootDir+
'/'):]
for p
in paths]
491 if pathStripped
is not None:
492 paths = [p + pathStripped
for p
in paths]
495 dir = os.path.join(dir,
"_parent")
496 if not os.path.exists(dir):
503 """Ask if a storage at the location described by uri exists 508 URI to the the root location of the storage 513 True if the storage exists, false if not 515 return os.path.exists(PosixStorage._pathFromURI(uri))
519 """Read an lsst.pex.config.Config from a butlerLocation. 523 butlerLocation : ButlerLocation 524 The location for the object(s) to be read. 528 A list of objects as described by the butler location. One item for 529 each location in butlerLocation.getLocations() 532 for locationString
in butlerLocation.getLocations():
533 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
534 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
535 if not os.path.exists(logLoc.locString()):
536 raise RuntimeError(
"No such config file: " + logLoc.locString())
537 pythonType = butlerLocation.getPythonType()
538 if pythonType
is not None:
539 if isinstance(pythonType, basestring):
541 finalItem = pythonType()
542 finalItem.load(logLoc.locString())
543 results.append(finalItem)
548 """Writes an lsst.pex.config.Config object to a location specified by 553 butlerLocation : ButlerLocation 554 The location for the object to be written. 555 obj : object instance 556 The object to be written. 558 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
560 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
561 obj.save(logLoc.locString())
565 """Read objects from a FITS file specified by ButlerLocation. 567 The object is read using class or static method 568 ``readFitsWithOptions(path, options)``, if it exists, else 569 ``readFits(path)``. The ``options`` argument is the data returned by 570 ``butlerLocation.getAdditionalData()``. 574 butlerLocation : ButlerLocation 575 The location for the object(s) to be read. 579 A list of objects as described by the butler location. One item for 580 each location in butlerLocation.getLocations() 582 pythonType = butlerLocation.getPythonType()
583 if pythonType
is not None:
584 if isinstance(pythonType, basestring):
586 supportsOptions = hasattr(pythonType,
"readFitsWithOptions")
588 additionalData = butlerLocation.getAdditionalData()
589 for locationString
in butlerLocation.getLocations():
590 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
594 filePath = re.sub(
r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$",
r"\1", logLoc.locString())
595 if not os.path.exists(filePath):
596 raise RuntimeError(
"No such FITS file: " + logLoc.locString())
598 finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
600 finalItem = pythonType.readFits(logLoc.locString())
601 results.append(finalItem)
606 """Writes an object to a FITS file specified by ButlerLocation. 608 The object is written using method 609 ``writeFitsWithOptions(path, options)``, if it exists, else 610 ``writeFits(path)``. The ``options`` argument is the data returned by 611 ``butlerLocation.getAdditionalData()``. 615 butlerLocation : ButlerLocation 616 The location for the object to be written. 617 obj : object instance 618 The object to be written. 620 supportsOptions = hasattr(obj,
"writeFitsWithOptions")
621 additionalData = butlerLocation.getAdditionalData()
622 locations = butlerLocation.getLocations()
623 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
626 obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
628 obj.writeFits(logLoc.locString())
632 """Read a catalog from a Parquet file specified by ButlerLocation. 634 The object returned by this is expected to be a subtype 635 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` 636 that allows for lazy loading of the data. 640 butlerLocation : ButlerLocation 641 The location for the object(s) to be read. 645 A list of objects as described by the butler location. One item for 646 each location in butlerLocation.getLocations() 649 additionalData = butlerLocation.getAdditionalData()
651 for locationString
in butlerLocation.getLocations():
652 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
654 if not os.path.exists(logLoc.locString()):
655 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
657 pythonType = butlerLocation.getPythonType()
658 if pythonType
is not None:
659 if isinstance(pythonType, basestring):
662 filename = logLoc.locString()
666 results.append(pythonType(filename=filename))
672 """Writes pandas dataframe to parquet file. 676 butlerLocation : ButlerLocation 677 The location for the object(s) to be read. 678 obj : `lsst.qa.explorer.parquetTable.ParquetTable` 679 Wrapped DataFrame to write. 682 additionalData = butlerLocation.getAdditionalData()
683 locations = butlerLocation.getLocations()
684 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
686 filename = logLoc.locString()
691 """Writes an object to a YAML file specified by ButlerLocation. 695 butlerLocation : ButlerLocation 696 The location for the object to be written. 697 obj : object instance 698 The object to be written. 700 additionalData = butlerLocation.getAdditionalData()
701 locations = butlerLocation.getLocations()
702 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
704 with open(logLoc.locString(),
"w")
as outfile:
705 yaml.dump(obj, outfile)
709 """Read an object from a pickle file specified by ButlerLocation. 713 butlerLocation : ButlerLocation 714 The location for the object(s) to be read. 718 A list of objects as described by the butler location. One item for 719 each location in butlerLocation.getLocations() 723 additionalData = butlerLocation.getAdditionalData()
724 for locationString
in butlerLocation.getLocations():
725 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
727 if not os.path.exists(logLoc.locString()):
728 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
729 with open(logLoc.locString(),
"rb")
as infile:
733 if sys.version_info.major >= 3:
734 finalItem = pickle.load(infile, encoding=
"latin1")
736 finalItem = pickle.load(infile)
737 results.append(finalItem)
742 """Writes an object to a pickle file specified by ButlerLocation. 746 butlerLocation : ButlerLocation 747 The location for the object to be written. 748 obj : object instance 749 The object to be written. 751 additionalData = butlerLocation.getAdditionalData()
752 locations = butlerLocation.getLocations()
753 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
755 with open(logLoc.locString(),
"wb")
as outfile:
756 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
760 """Read a catalog from a FITS table specified by ButlerLocation. 764 butlerLocation : ButlerLocation 765 The location for the object(s) to be read. 769 A list of objects as described by the butler location. One item for 770 each location in butlerLocation.getLocations() 772 pythonType = butlerLocation.getPythonType()
773 if pythonType
is not None:
774 if isinstance(pythonType, basestring):
777 additionalData = butlerLocation.getAdditionalData()
778 for locationString
in butlerLocation.getLocations():
779 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
781 if not os.path.exists(logLoc.locString()):
782 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
784 if additionalData.exists(
"hdu"):
785 kwds[
"hdu"] = additionalData.getInt(
"hdu")
786 if additionalData.exists(
"flags"):
787 kwds[
"flags"] = additionalData.getInt(
"flags")
788 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
789 results.append(finalItem)
794 """Writes a catalog to a FITS table specified by ButlerLocation. 798 butlerLocation : ButlerLocation 799 The location for the object to be written. 800 obj : object instance 801 The object to be written. 803 additionalData = butlerLocation.getAdditionalData()
804 locations = butlerLocation.getLocations()
805 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
807 if additionalData.exists(
"flags"):
808 kwds = dict(flags=additionalData.getInt(
"flags"))
811 obj.writeFits(logLoc.locString(), **kwds)
815 """Read from a butlerLocation (always fails for this storage type). 819 butlerLocation : ButlerLocation 820 The location for the object(s) to be read. 824 A list of objects as described by the butler location. One item for 825 each location in butlerLocation.getLocations() 827 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
831 """Writes a matplotlib.figure.Figure to a location, using the template's 832 filename suffix to infer the file format. 836 butlerLocation : ButlerLocation 837 The location for the object to be written. 838 obj : matplotlib.figure.Figure 839 The object to be written. 841 additionalData = butlerLocation.getAdditionalData()
842 locations = butlerLocation.getLocations()
843 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
849 _, ext = os.path.splitext(locations[0])
856 obj.savefig(logLoc.locString(), format=ext)
860 """Read a policy from a PAF file specified by a ButlerLocation. 864 butlerLocation : ButlerLocation 865 The location for the object(s) to be read. 869 A list of objects as described by the butler location. One item for 870 each location in butlerLocation.getLocations() 873 for locationString
in butlerLocation.getLocations():
874 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
875 butlerLocation.getAdditionalData())
876 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
877 results.append(finalItem)
882 """Read an object from a YAML file specified by a butlerLocation. 886 butlerLocation : ButlerLocation 887 The location for the object(s) to be read. 891 A list of objects as described by the butler location. One item for 892 each location in butlerLocation.getLocations() 895 for locationString
in butlerLocation.getLocations():
896 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
897 butlerLocation.getAdditionalData())
898 if not os.path.exists(logLoc.locString()):
899 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
901 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
902 finalItem =
Policy(filePath=logLoc.locString())
904 with open(logLoc.locString(),
"rb")
as infile:
905 finalItem = yaml.load(infile)
906 results.append(finalItem)
911 """Read an object from a boost::serialization file. 915 butlerLocation : ButlerLocation 916 The location for the object(s) to be read. 920 A list of objects as described by the butler location. One item for 921 each location in butlerLocation.getLocations() 924 additionalData = butlerLocation.getAdditionalData()
925 for locationString
in butlerLocation.getLocations():
926 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
927 butlerLocation.getAdditionalData())
929 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
930 storageList.append(storage)
931 finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
933 results.append(finalItem)
938 """Writes an object via boost::serialization. 942 butlerLocation : ButlerLocation 943 The location for the object to be written. 944 obj : object instance 945 The object to be written. 947 additionalData = butlerLocation.getAdditionalData()
948 location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
953 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
954 storageList.append(storage)
956 if hasattr(obj,
'__deref__'):
958 PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
960 PosixStorage.getPersistence().persist(obj, storageList, additionalData)
963 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
964 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
965 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
966 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
967 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
968 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
969 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
970 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
972 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
973 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def safeMakeDir(directory)
def writeMatplotlibStorage(butlerLocation, obj)
def getWriteFormatter(cls, objType)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def relativePath(fromPath, toPath)
def writeFitsCatalogStorage(butlerLocation, obj)
def exists(self, location)
def readPafStorage(butlerLocation)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def locationWithRoot(self, location)
def readFitsCatalogStorage(butlerLocation)
def getParentSymlinkPath(root)
def readYamlStorage(butlerLocation)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getReadFormatter(cls, objType)
def writeYamlStorage(butlerLocation, obj)
def getLocalFile(self, path)
def read(self, butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)