24 from past.builtins
import basestring
34 from .
import (LogicalLocation, Persistence, Policy, StorageList,
35 StorageInterface, Storage, ButlerLocation,
36 NoRepositroyAtRoot, RepositoryCfg, doImport)
39 from .safeFileIo
import SafeFilename, safeMakeDir
41 from .
import baseYaml
44 __all__ = [
"PosixStorage"]
48 """Defines the interface for a storage location on the local filesystem. 53 URI or path that is used as the storage location. 55 If True a new repository will be created at the root location if it 56 does not exist. If False then a new repository will not be created. 61 If create is False and a repository does not exist at the root 62 specified by uri then NoRepositroyAtRoot is raised. 66 self.
log = Log.getLogger(
"daf.persistence.butler")
68 if self.
root and not os.path.exists(self.
root):
79 return Persistence.getPersistence(persistencePolicy)
82 return 'PosixStorage(root=%s)' % self.
root 85 def _pathFromURI(uri):
86 """Get the path part of the URI""" 87 return urllib.parse.urlparse(uri).path
91 """Get a relative path from a location to a location. 96 A path at which to start. It can be a relative path or an 99 A target location. It can be a relative path or an absolute path. 104 A relative path that describes the path from fromPath to toPath. 106 fromPath = os.path.realpath(fromPath)
107 return os.path.relpath(toPath, fromPath)
111 """Get an absolute path for the path from fromUri to toUri 115 fromPath : the starting location 116 A location at which to start. It can be a relative path or an 118 relativePath : the location relative to fromPath 124 Path that is an absolute path representation of fromPath + 125 relativePath, if one exists. If relativePath is absolute or if 126 fromPath is not related to relativePath then relativePath will be 129 if os.path.isabs(relativePath):
131 fromPath = os.path.realpath(fromPath)
132 return os.path.normpath(os.path.join(fromPath, relativePath))
136 """Get a persisted RepositoryCfg 140 uri : URI or path to a RepositoryCfg 145 A RepositoryCfg instance or None 147 storage = Storage.makeFromURI(uri)
151 locationList=
'repositoryCfg.yaml',
157 return storage.read(location)
161 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
165 locationList=
'repositoryCfg.yaml',
171 storage.write(location, cfg)
175 """Get the mapper class associated with a repository root. 177 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 178 new code and repositories; they should use the Repository parentCfg mechanism. 183 The location of a persisted ReositoryCfg is (new style repos), or 184 the location where a _mapper file is (old style repos). 188 A class object or a class instance, depending on the state of the 189 mapper when the repository was created. 194 cfg = PosixStorage.getRepositoryCfg(root)
200 mapperFile =
"_mapper" 201 while not os.path.exists(os.path.join(basePath, mapperFile)):
203 if os.path.exists(os.path.join(basePath,
"_parent")):
204 basePath = os.path.join(basePath,
"_parent")
209 if mapperFile
is not None:
210 mapperFile = os.path.join(basePath, mapperFile)
213 with open(mapperFile,
"r") as f: 214 mapperName = f.readline().strip() 215 components = mapperName.split(".")
216 if len(components) <= 1:
217 raise RuntimeError(
"Unqualified mapper name %s in %s" %
218 (mapperName, mapperFile))
219 pkg = importlib.import_module(
".".join(components[:-1]))
220 return getattr(pkg, components[-1])
226 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 232 A path to the folder on the local filesystem. 237 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 240 linkpath = os.path.join(root,
'_parent')
241 if os.path.exists(linkpath):
243 return os.readlink(os.path.join(root,
'_parent'))
247 return os.path.join(root,
'_parent')
250 def write(self, butlerLocation, obj):
251 """Writes an object to a location and persistence format specified by 256 butlerLocation : ButlerLocation 257 The location & formatting for the object to be written. 258 obj : object instance 259 The object to be written. 261 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
264 if not writeFormatter:
267 writeFormatter(butlerLocation, obj)
270 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
272 def read(self, butlerLocation):
273 """Read from a butlerLocation. 277 butlerLocation : ButlerLocation 278 The location & formatting for the object(s) to be read. 282 A list of objects as described by the butler location. One item for 283 each location in butlerLocation.getLocations() 286 if not readFormatter:
289 return readFormatter(butlerLocation)
291 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
294 """Implementation of PosixStorage.exists for ButlerLocation objects. 296 storageName = location.getStorageName()
297 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
298 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
299 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
300 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
302 for locationString
in location.getLocations():
303 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
310 """Check if location exists. 314 location : ButlerLocation or string 315 A a string or a ButlerLocation that describes the location of an 316 object in this storage. 321 True if exists, else False. 323 if isinstance(location, ButlerLocation):
330 """Get the full path to the location. 335 return os.path.join(self.
root, location)
339 """Test if a Version 1 Repository exists. 341 Version 1 Repositories only exist in posix storages, do not have a 342 RepositoryCfg file, and contain either a registry.sqlite3 file, a 343 _mapper file, or a _parent link. 348 A path to a folder on the local filesystem. 353 True if the repository at root exists, else False. 355 return os.path.exists(root)
and (
356 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or 357 os.path.exists(os.path.join(root,
"_mapper"))
or 358 os.path.exists(os.path.join(root,
"_parent"))
362 """Copy a file from one location to another on the local filesystem. 367 Path and name of existing file. 369 Path and name of new file. 375 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
378 """Get a handle to a local copy of the file, downloading it to a 383 A path the the file in storage, relative to root. 387 A handle to a local copy of the file. If storage is remote it will be 388 a temporary file. If storage is local it may be the original file or 389 a temporary file. The file name can be gotten via the 'name' property 390 of the returned object. 392 p = os.path.join(self.
root, path)
402 """Search for the given path in this storage instance. 404 If the path contains an HDU indicator (a number in brackets before the 405 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 406 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 407 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 412 A filename (and optionally prefix path) to search for within root. 417 The location that was found, or None if no location was found. 422 def search(root, path, searchParents=False):
423 """Look for the given path in the current root. 425 Also supports searching for the path in Butler v1 repositories by 426 following the Butler v1 _parent symlink 428 If the path contains an HDU indicator (a number in brackets, e.g. 429 'foo.fits[1]', this will be stripped when searching and so 430 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 431 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 436 The path to the root directory. 438 The path to the file within the root directory. 439 searchParents : bool, optional 440 For Butler v1 repositories only, if true and a _parent symlink 441 exists, then the directory at _parent will be searched if the file 442 is not found in the root repository. Will continue searching the 443 parent of the parent until the file is found or no additional 449 The location that was found, or None if no location was found. 455 while len(rootDir) > 1
and rootDir[-1] ==
'/':
456 rootDir = rootDir[:-1]
458 if path.startswith(rootDir +
"/"):
460 path = path[len(rootDir +
'/'):]
462 elif rootDir ==
"/" and path.startswith(
"/"):
467 pathPrefix = os.path.dirname(path)
468 while pathPrefix !=
"" and pathPrefix !=
"/":
469 if os.path.realpath(pathPrefix) == os.path.realpath(root):
471 pathPrefix = os.path.dirname(pathPrefix)
472 if pathPrefix ==
"/":
474 elif pathPrefix !=
"":
475 path = path[len(pathPrefix)+1:]
481 firstBracket = path.find(
"[")
482 if firstBracket != -1:
483 strippedPath = path[:firstBracket]
484 pathStripped = path[firstBracket:]
488 paths = glob.glob(os.path.join(dir, strippedPath))
490 if pathPrefix != rootDir:
491 paths = [p[len(rootDir+
'/'):]
for p
in paths]
492 if pathStripped
is not None:
493 paths = [p + pathStripped
for p
in paths]
496 dir = os.path.join(dir,
"_parent")
497 if not os.path.exists(dir):
504 """Ask if a storage at the location described by uri exists 509 URI to the the root location of the storage 514 True if the storage exists, false if not 516 return os.path.exists(PosixStorage._pathFromURI(uri))
520 """Read an lsst.pex.config.Config from a butlerLocation. 524 butlerLocation : ButlerLocation 525 The location for the object(s) to be read. 529 A list of objects as described by the butler location. One item for 530 each location in butlerLocation.getLocations() 533 for locationString
in butlerLocation.getLocations():
534 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
535 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
536 if not os.path.exists(logLoc.locString()):
537 raise RuntimeError(
"No such config file: " + logLoc.locString())
538 pythonType = butlerLocation.getPythonType()
539 if pythonType
is not None:
540 if isinstance(pythonType, basestring):
542 finalItem = pythonType()
543 finalItem.load(logLoc.locString())
544 results.append(finalItem)
549 """Writes an lsst.pex.config.Config object to a location specified by 554 butlerLocation : ButlerLocation 555 The location for the object to be written. 556 obj : object instance 557 The object to be written. 559 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
561 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
562 obj.save(logLoc.locString())
566 """Read a FITS image from a butlerLocation. 570 butlerLocation : ButlerLocation 571 The location for the object(s) to be read. 575 A list of objects as described by the butler location. One item for 576 each location in butlerLocation.getLocations() 579 for locationString
in butlerLocation.getLocations():
580 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
581 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
583 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
585 storageList.append(storage)
586 finalItem = PosixStorage.getPersistence().unsafeRetrieve(
587 butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
588 results.append(finalItem)
593 """Writes an object to a FITS file specified by ButlerLocation. 597 butlerLocation : ButlerLocation 598 The location for the object to be written. 599 obj : object instance 600 The object to be written. 602 location = butlerLocation.getLocations()[0]
603 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, location))
as locationString:
604 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
607 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
608 storageList.append(storage)
609 persistence = PosixStorage.getPersistence()
610 if hasattr(obj,
'__deref__'):
612 persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
614 persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
618 """Read a catalog from a Parquet file specified by ButlerLocation. 620 The object returned by this is expected to be a subtype 621 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` 622 that allows for lazy loading of the data. 626 butlerLocation : ButlerLocation 627 The location for the object(s) to be read. 631 A list of objects as described by the butler location. One item for 632 each location in butlerLocation.getLocations() 635 additionalData = butlerLocation.getAdditionalData()
637 for locationString
in butlerLocation.getLocations():
638 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
640 if not os.path.exists(logLoc.locString()):
641 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
643 pythonType = butlerLocation.getPythonType()
644 if pythonType
is not None:
645 if isinstance(pythonType, basestring):
648 filename = logLoc.locString()
652 results.append(pythonType(filename=filename))
658 """Writes pandas dataframe to parquet file. 662 butlerLocation : ButlerLocation 663 The location for the object(s) to be read. 664 obj : `lsst.qa.explorer.parquetTable.ParquetTable` 665 Wrapped DataFrame to write. 668 additionalData = butlerLocation.getAdditionalData()
669 locations = butlerLocation.getLocations()
670 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
672 filename = logLoc.locString()
677 """Writes an object to a YAML file specified by ButlerLocation. 681 butlerLocation : ButlerLocation 682 The location for the object to be written. 683 obj : object instance 684 The object to be written. 686 additionalData = butlerLocation.getAdditionalData()
687 locations = butlerLocation.getLocations()
688 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
690 with open(logLoc.locString(),
"w")
as outfile:
691 yaml.dump(obj, outfile)
695 """Read an object from a pickle file specified by ButlerLocation. 699 butlerLocation : ButlerLocation 700 The location for the object(s) to be read. 704 A list of objects as described by the butler location. One item for 705 each location in butlerLocation.getLocations() 709 additionalData = butlerLocation.getAdditionalData()
710 for locationString
in butlerLocation.getLocations():
711 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
713 if not os.path.exists(logLoc.locString()):
714 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
715 with open(logLoc.locString(),
"rb")
as infile:
719 if sys.version_info.major >= 3:
720 finalItem = pickle.load(infile, encoding=
"latin1")
722 finalItem = pickle.load(infile)
723 results.append(finalItem)
728 """Writes an object to a pickle file specified by ButlerLocation. 732 butlerLocation : ButlerLocation 733 The location for the object to be written. 734 obj : object instance 735 The object to be written. 737 additionalData = butlerLocation.getAdditionalData()
738 locations = butlerLocation.getLocations()
739 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
741 with open(logLoc.locString(),
"wb")
as outfile:
742 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
746 """Read a catalog from a FITS table specified by ButlerLocation. 750 butlerLocation : ButlerLocation 751 The location for the object(s) to be read. 755 A list of objects as described by the butler location. One item for 756 each location in butlerLocation.getLocations() 758 pythonType = butlerLocation.getPythonType()
759 if pythonType
is not None:
760 if isinstance(pythonType, basestring):
763 additionalData = butlerLocation.getAdditionalData()
764 for locationString
in butlerLocation.getLocations():
765 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
767 if not os.path.exists(logLoc.locString()):
768 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
770 if additionalData.exists(
"hdu"):
771 kwds[
"hdu"] = additionalData.getInt(
"hdu")
772 if additionalData.exists(
"flags"):
773 kwds[
"flags"] = additionalData.getInt(
"flags")
774 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
775 results.append(finalItem)
780 """Writes a catalog to a FITS table specified by ButlerLocation. 784 butlerLocation : ButlerLocation 785 The location for the object to be written. 786 obj : object instance 787 The object to be written. 789 additionalData = butlerLocation.getAdditionalData()
790 locations = butlerLocation.getLocations()
791 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
793 if additionalData.exists(
"flags"):
794 kwds = dict(flags=additionalData.getInt(
"flags"))
797 obj.writeFits(logLoc.locString(), **kwds)
801 """Read from a butlerLocation (always fails for this storage type). 805 butlerLocation : ButlerLocation 806 The location for the object(s) to be read. 810 A list of objects as described by the butler location. One item for 811 each location in butlerLocation.getLocations() 813 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
817 """Writes a matplotlib.figure.Figure to a location, using the template's 818 filename suffix to infer the file format. 822 butlerLocation : ButlerLocation 823 The location for the object to be written. 824 obj : matplotlib.figure.Figure 825 The object to be written. 827 additionalData = butlerLocation.getAdditionalData()
828 locations = butlerLocation.getLocations()
829 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
835 _, ext = os.path.splitext(locations[0])
842 obj.savefig(logLoc.locString(), format=ext)
846 """Read a policy from a PAF file specified by a ButlerLocation. 850 butlerLocation : ButlerLocation 851 The location for the object(s) to be read. 855 A list of objects as described by the butler location. One item for 856 each location in butlerLocation.getLocations() 859 for locationString
in butlerLocation.getLocations():
860 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
861 butlerLocation.getAdditionalData())
862 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
863 results.append(finalItem)
868 """Read an object from a YAML file specified by a butlerLocation. 872 butlerLocation : ButlerLocation 873 The location for the object(s) to be read. 877 A list of objects as described by the butler location. One item for 878 each location in butlerLocation.getLocations() 881 for locationString
in butlerLocation.getLocations():
882 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
883 butlerLocation.getAdditionalData())
884 if not os.path.exists(logLoc.locString()):
885 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
887 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
888 finalItem =
Policy(filePath=logLoc.locString())
890 with open(logLoc.locString(),
"rb")
as infile:
891 finalItem = yaml.load(infile)
892 results.append(finalItem)
897 """Read an object from a boost::serialization file. 901 butlerLocation : ButlerLocation 902 The location for the object(s) to be read. 906 A list of objects as described by the butler location. One item for 907 each location in butlerLocation.getLocations() 910 additionalData = butlerLocation.getAdditionalData()
911 for locationString
in butlerLocation.getLocations():
912 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
913 butlerLocation.getAdditionalData())
915 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
916 storageList.append(storage)
917 finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
919 results.append(finalItem)
924 """Writes an object via boost::serialization. 928 butlerLocation : ButlerLocation 929 The location for the object to be written. 930 obj : object instance 931 The object to be written. 933 additionalData = butlerLocation.getAdditionalData()
934 location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
939 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
940 storageList.append(storage)
942 if hasattr(obj,
'__deref__'):
944 PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
946 PosixStorage.getPersistence().persist(obj, storageList, additionalData)
949 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
950 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
951 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
952 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
953 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
954 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
955 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
956 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
957 PosixStorage.registerFormatters(
"BoostStorage", readFitsStorage, writeFitsStorage)
959 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
960 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def safeMakeDir(directory)
def writeMatplotlibStorage(butlerLocation, obj)
def getWriteFormatter(cls, objType)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def relativePath(fromPath, toPath)
def writeFitsCatalogStorage(butlerLocation, obj)
def exists(self, location)
def readPafStorage(butlerLocation)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def locationWithRoot(self, location)
def readFitsCatalogStorage(butlerLocation)
def getParentSymlinkPath(root)
def readYamlStorage(butlerLocation)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getReadFormatter(cls, objType)
def writeYamlStorage(butlerLocation, obj)
def getLocalFile(self, path)
def read(self, butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)