24 from past.builtins
import basestring
35 from .
import (LogicalLocation, Persistence, Policy, StorageList,
36 StorageInterface, Storage, ButlerLocation,
37 NoRepositroyAtRoot, RepositoryCfg, doImport)
40 from .safeFileIo
import SafeFilename, safeMakeDir
43 __all__ = [
"PosixStorage"]
47 """Defines the interface for a storage location on the local filesystem. 52 URI or path that is used as the storage location. 54 If True a new repository will be created at the root location if it 55 does not exist. If False then a new repository will not be created. 60 If create is False and a repository does not exist at the root 61 specified by uri then NoRepositroyAtRoot is raised. 65 self.
log = Log.getLogger(
"daf.persistence.butler")
67 if self.
root and not os.path.exists(self.
root):
78 return Persistence.getPersistence(persistencePolicy)
81 return 'PosixStorage(root=%s)' % self.
root 84 def _pathFromURI(uri):
85 """Get the path part of the URI""" 86 return urllib.parse.urlparse(uri).path
90 """Get a relative path from a location to a location. 95 A path at which to start. It can be a relative path or an 98 A target location. It can be a relative path or an absolute path. 103 A relative path that describes the path from fromPath to toPath. 105 fromPath = os.path.realpath(fromPath)
106 return os.path.relpath(toPath, fromPath)
110 """Get an absolute path for the path from fromUri to toUri 114 fromPath : the starting location 115 A location at which to start. It can be a relative path or an 117 relativePath : the location relative to fromPath 123 Path that is an absolute path representation of fromPath + 124 relativePath, if one exists. If relativePath is absolute or if 125 fromPath is not related to relativePath then relativePath will be 128 if os.path.isabs(relativePath):
130 fromPath = os.path.realpath(fromPath)
131 return os.path.normpath(os.path.join(fromPath, relativePath))
135 """Get a persisted RepositoryCfg 139 uri : URI or path to a RepositoryCfg 144 A RepositoryCfg instance or None 146 storage = Storage.makeFromURI(uri)
150 locationList=
'repositoryCfg.yaml',
156 return storage.read(location)
160 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
164 locationList=
'repositoryCfg.yaml',
170 storage.write(location, cfg)
174 """Get the mapper class associated with a repository root. 176 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 177 new code and repositories; they should use the Repository parentCfg mechanism. 182 The location of a persisted ReositoryCfg is (new style repos), or 183 the location where a _mapper file is (old style repos). 187 A class object or a class instance, depending on the state of the 188 mapper when the repository was created. 193 cfg = PosixStorage.getRepositoryCfg(root)
199 mapperFile =
"_mapper" 200 while not os.path.exists(os.path.join(basePath, mapperFile)):
202 if os.path.exists(os.path.join(basePath,
"_parent")):
203 basePath = os.path.join(basePath,
"_parent")
208 if mapperFile
is not None:
209 mapperFile = os.path.join(basePath, mapperFile)
212 with open(mapperFile,
"r") as f: 213 mapperName = f.readline().strip() 214 components = mapperName.split(".")
215 if len(components) <= 1:
216 raise RuntimeError(
"Unqualified mapper name %s in %s" %
217 (mapperName, mapperFile))
218 pkg = importlib.import_module(
".".join(components[:-1]))
219 return getattr(pkg, components[-1])
225 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 231 A path to the folder on the local filesystem. 236 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 239 linkpath = os.path.join(root,
'_parent')
240 if os.path.exists(linkpath):
242 return os.readlink(os.path.join(root,
'_parent'))
246 return os.path.join(root,
'_parent')
249 def write(self, butlerLocation, obj):
250 """Writes an object to a location and persistence format specified by 255 butlerLocation : ButlerLocation 256 The location & formatting for the object to be written. 257 obj : object instance 258 The object to be written. 260 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
263 if not writeFormatter:
266 writeFormatter(butlerLocation, obj)
269 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
271 def read(self, butlerLocation):
272 """Read from a butlerLocation. 276 butlerLocation : ButlerLocation 277 The location & formatting for the object(s) to be read. 281 A list of objects as described by the butler location. One item for 282 each location in butlerLocation.getLocations() 285 if not readFormatter:
288 return readFormatter(butlerLocation)
290 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
293 """Implementation of PosixStorage.exists for ButlerLocation objects. 295 storageName = location.getStorageName()
296 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
297 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
298 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
299 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
301 for locationString
in location.getLocations():
302 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
309 """Check if location exists. 313 location : ButlerLocation or string 314 A a string or a ButlerLocation that describes the location of an 315 object in this storage. 320 True if exists, else False. 322 if isinstance(location, ButlerLocation):
329 """Get the full path to the location. 334 return os.path.join(self.
root, location)
338 """Test if a Version 1 Repository exists. 340 Version 1 Repositories only exist in posix storages, do not have a 341 RepositoryCfg file, and contain either a registry.sqlite3 file, a 342 _mapper file, or a _parent link. 347 A path to a folder on the local filesystem. 352 True if the repository at root exists, else False. 354 return os.path.exists(root)
and (
355 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or 356 os.path.exists(os.path.join(root,
"_mapper"))
or 357 os.path.exists(os.path.join(root,
"_parent"))
361 """Copy a file from one location to another on the local filesystem. 366 Path and name of existing file. 368 Path and name of new file. 374 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
377 """Get a handle to a local copy of the file, downloading it to a 382 A path the the file in storage, relative to root. 386 A handle to a local copy of the file. If storage is remote it will be 387 a temporary file. If storage is local it may be the original file or 388 a temporary file. The file name can be gotten via the 'name' property 389 of the returned object. 391 p = os.path.join(self.
root, path)
401 """Search for the given path in this storage instance. 403 If the path contains an HDU indicator (a number in brackets before the 404 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 405 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 406 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 411 A filename (and optionally prefix path) to search for within root. 416 The location that was found, or None if no location was found. 421 def search(root, path, searchParents=False):
422 """Look for the given path in the current root. 424 Also supports searching for the path in Butler v1 repositories by 425 following the Butler v1 _parent symlink 427 If the path contains an HDU indicator (a number in brackets, e.g. 428 'foo.fits[1]', this will be stripped when searching and so 429 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 430 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 435 The path to the root directory. 437 The path to the file within the root directory. 438 searchParents : bool, optional 439 For Butler v1 repositories only, if true and a _parent symlink 440 exists, then the directory at _parent will be searched if the file 441 is not found in the root repository. Will continue searching the 442 parent of the parent until the file is found or no additional 448 The location that was found, or None if no location was found. 454 while len(rootDir) > 1
and rootDir[-1] ==
'/':
455 rootDir = rootDir[:-1]
457 if path.startswith(rootDir +
"/"):
459 path = path[len(rootDir +
'/'):]
461 elif rootDir ==
"/" and path.startswith(
"/"):
466 pathPrefix = os.path.dirname(path)
467 while pathPrefix !=
"" and pathPrefix !=
"/":
468 if os.path.realpath(pathPrefix) == os.path.realpath(root):
470 pathPrefix = os.path.dirname(pathPrefix)
471 if pathPrefix ==
"/":
473 elif pathPrefix !=
"":
474 path = path[len(pathPrefix)+1:]
480 firstBracket = path.find(
"[")
481 if firstBracket != -1:
482 strippedPath = path[:firstBracket]
483 pathStripped = path[firstBracket:]
487 paths = glob.glob(os.path.join(dir, strippedPath))
489 if pathPrefix != rootDir:
490 paths = [p[len(rootDir+
'/'):]
for p
in paths]
491 if pathStripped
is not None:
492 paths = [p + pathStripped
for p
in paths]
495 dir = os.path.join(dir,
"_parent")
496 if not os.path.exists(dir):
503 """Ask if a storage at the location described by uri exists 508 URI to the the root location of the storage 513 True if the storage exists, false if not 515 return os.path.exists(PosixStorage._pathFromURI(uri))
519 """Read an lsst.pex.config.Config from a butlerLocation. 523 butlerLocation : ButlerLocation 524 The location for the object(s) to be read. 528 A list of objects as described by the butler location. One item for 529 each location in butlerLocation.getLocations() 532 for locationString
in butlerLocation.getLocations():
533 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
534 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
535 if not os.path.exists(logLoc.locString()):
536 raise RuntimeError(
"No such config file: " + logLoc.locString())
537 pythonType = butlerLocation.getPythonType()
538 if pythonType
is not None:
539 if isinstance(pythonType, basestring):
540 pythonType = doImport(pythonType)
541 finalItem = pythonType()
542 finalItem.load(logLoc.locString())
543 results.append(finalItem)
548 """Writes an lsst.pex.config.Config object to a location specified by 553 butlerLocation : ButlerLocation 554 The location for the object to be written. 555 obj : object instance 556 The object to be written. 558 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
560 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
561 obj.save(logLoc.locString())
565 """Read objects from a FITS file specified by ButlerLocation. 567 The object is read using class or static method 568 ``readFitsWithOptions(path, options)``, if it exists, else 569 ``readFits(path)``. The ``options`` argument is the data returned by 570 ``butlerLocation.getAdditionalData()``. 574 butlerLocation : ButlerLocation 575 The location for the object(s) to be read. 579 A list of objects as described by the butler location. One item for 580 each location in butlerLocation.getLocations() 582 pythonType = butlerLocation.getPythonType()
583 if pythonType
is not None:
584 if isinstance(pythonType, basestring):
585 pythonType = doImport(pythonType)
586 supportsOptions = hasattr(pythonType,
"readFitsWithOptions")
587 if not supportsOptions:
589 if issubclass(pythonType, (PropertySet, PropertyList)):
590 from lsst.afw.image
import readMetadata
591 reader = readMetadata
593 reader = pythonType.readFits
595 additionalData = butlerLocation.getAdditionalData()
596 for locationString
in butlerLocation.getLocations():
597 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
601 filePath = re.sub(
r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$",
r"\1", logLoc.locString())
602 if not os.path.exists(filePath):
603 raise RuntimeError(
"No such FITS file: " + logLoc.locString())
605 finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
607 finalItem = reader(logLoc.locString())
608 results.append(finalItem)
613 """Writes an object to a FITS file specified by ButlerLocation. 615 The object is written using method 616 ``writeFitsWithOptions(path, options)``, if it exists, else 617 ``writeFits(path)``. The ``options`` argument is the data returned by 618 ``butlerLocation.getAdditionalData()``. 622 butlerLocation : ButlerLocation 623 The location for the object to be written. 624 obj : object instance 625 The object to be written. 627 supportsOptions = hasattr(obj,
"writeFitsWithOptions")
628 additionalData = butlerLocation.getAdditionalData()
629 locations = butlerLocation.getLocations()
630 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
633 obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
635 obj.writeFits(logLoc.locString())
639 """Read a catalog from a Parquet file specified by ButlerLocation. 641 The object returned by this is expected to be a subtype 642 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` 643 that allows for lazy loading of the data. 647 butlerLocation : ButlerLocation 648 The location for the object(s) to be read. 652 A list of objects as described by the butler location. One item for 653 each location in butlerLocation.getLocations() 656 additionalData = butlerLocation.getAdditionalData()
658 for locationString
in butlerLocation.getLocations():
659 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
661 if not os.path.exists(logLoc.locString()):
662 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
664 pythonType = butlerLocation.getPythonType()
665 if pythonType
is not None:
666 if isinstance(pythonType, basestring):
667 pythonType = doImport(pythonType)
669 filename = logLoc.locString()
673 results.append(pythonType(filename=filename))
679 """Writes pandas dataframe to parquet file. 683 butlerLocation : ButlerLocation 684 The location for the object(s) to be read. 685 obj : `lsst.qa.explorer.parquetTable.ParquetTable` 686 Wrapped DataFrame to write. 689 additionalData = butlerLocation.getAdditionalData()
690 locations = butlerLocation.getLocations()
691 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
693 filename = logLoc.locString()
698 """Writes an object to a YAML file specified by ButlerLocation. 702 butlerLocation : ButlerLocation 703 The location for the object to be written. 704 obj : object instance 705 The object to be written. 707 additionalData = butlerLocation.getAdditionalData()
708 locations = butlerLocation.getLocations()
709 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
711 with open(logLoc.locString(),
"w")
as outfile:
712 yaml.dump(obj, outfile)
716 """Read an object from a pickle file specified by ButlerLocation. 720 butlerLocation : ButlerLocation 721 The location for the object(s) to be read. 725 A list of objects as described by the butler location. One item for 726 each location in butlerLocation.getLocations() 730 additionalData = butlerLocation.getAdditionalData()
731 for locationString
in butlerLocation.getLocations():
732 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
734 if not os.path.exists(logLoc.locString()):
735 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
736 with open(logLoc.locString(),
"rb")
as infile:
740 if sys.version_info.major >= 3:
741 finalItem = pickle.load(infile, encoding=
"latin1")
743 finalItem = pickle.load(infile)
744 results.append(finalItem)
749 """Writes an object to a pickle file specified by ButlerLocation. 753 butlerLocation : ButlerLocation 754 The location for the object to be written. 755 obj : object instance 756 The object to be written. 758 additionalData = butlerLocation.getAdditionalData()
759 locations = butlerLocation.getLocations()
760 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
762 with open(logLoc.locString(),
"wb")
as outfile:
763 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
767 """Read a catalog from a FITS table specified by ButlerLocation. 771 butlerLocation : ButlerLocation 772 The location for the object(s) to be read. 776 A list of objects as described by the butler location. One item for 777 each location in butlerLocation.getLocations() 779 pythonType = butlerLocation.getPythonType()
780 if pythonType
is not None:
781 if isinstance(pythonType, basestring):
782 pythonType = doImport(pythonType)
784 additionalData = butlerLocation.getAdditionalData()
785 for locationString
in butlerLocation.getLocations():
786 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
788 if not os.path.exists(logLoc.locString()):
789 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
791 if additionalData.exists(
"hdu"):
792 kwds[
"hdu"] = additionalData.getInt(
"hdu")
793 if additionalData.exists(
"flags"):
794 kwds[
"flags"] = additionalData.getInt(
"flags")
795 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
796 results.append(finalItem)
801 """Writes a catalog to a FITS table specified by ButlerLocation. 805 butlerLocation : ButlerLocation 806 The location for the object to be written. 807 obj : object instance 808 The object to be written. 810 additionalData = butlerLocation.getAdditionalData()
811 locations = butlerLocation.getLocations()
812 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
814 if additionalData.exists(
"flags"):
815 kwds = dict(flags=additionalData.getInt(
"flags"))
818 obj.writeFits(logLoc.locString(), **kwds)
822 """Read from a butlerLocation (always fails for this storage type). 826 butlerLocation : ButlerLocation 827 The location for the object(s) to be read. 831 A list of objects as described by the butler location. One item for 832 each location in butlerLocation.getLocations() 834 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
838 """Writes a matplotlib.figure.Figure to a location, using the template's 839 filename suffix to infer the file format. 843 butlerLocation : ButlerLocation 844 The location for the object to be written. 845 obj : matplotlib.figure.Figure 846 The object to be written. 848 additionalData = butlerLocation.getAdditionalData()
849 locations = butlerLocation.getLocations()
850 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
856 _, ext = os.path.splitext(locations[0])
863 obj.savefig(logLoc.locString(), format=ext)
867 """Read a policy from a PAF file specified by a ButlerLocation. 871 butlerLocation : ButlerLocation 872 The location for the object(s) to be read. 876 A list of objects as described by the butler location. One item for 877 each location in butlerLocation.getLocations() 880 for locationString
in butlerLocation.getLocations():
881 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
882 butlerLocation.getAdditionalData())
883 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
884 results.append(finalItem)
889 """Read an object from a YAML file specified by a butlerLocation. 893 butlerLocation : ButlerLocation 894 The location for the object(s) to be read. 898 A list of objects as described by the butler location. One item for 899 each location in butlerLocation.getLocations() 902 for locationString
in butlerLocation.getLocations():
903 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
904 butlerLocation.getAdditionalData())
905 if not os.path.exists(logLoc.locString()):
906 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
908 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
909 finalItem =
Policy(filePath=logLoc.locString())
911 with open(logLoc.locString(),
"rb")
as infile:
912 finalItem = yaml.load(infile)
913 results.append(finalItem)
918 """Read an object from a boost::serialization file. 922 butlerLocation : ButlerLocation 923 The location for the object(s) to be read. 927 A list of objects as described by the butler location. One item for 928 each location in butlerLocation.getLocations() 931 additionalData = butlerLocation.getAdditionalData()
932 for locationString
in butlerLocation.getLocations():
933 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
934 butlerLocation.getAdditionalData())
936 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
937 storageList.append(storage)
938 finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
940 results.append(finalItem)
945 """Writes an object via boost::serialization. 949 butlerLocation : ButlerLocation 950 The location for the object to be written. 951 obj : object instance 952 The object to be written. 954 additionalData = butlerLocation.getAdditionalData()
955 location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
960 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
961 storageList.append(storage)
963 if hasattr(obj,
'__deref__'):
965 PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
967 PosixStorage.getPersistence().persist(obj, storageList, additionalData)
970 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
971 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
972 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
973 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
974 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
975 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
976 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
977 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
979 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
980 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def safeMakeDir(directory)
def writeMatplotlibStorage(butlerLocation, obj)
def getWriteFormatter(cls, objType)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def relativePath(fromPath, toPath)
def writeFitsCatalogStorage(butlerLocation, obj)
def exists(self, location)
def readPafStorage(butlerLocation)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def locationWithRoot(self, location)
def readFitsCatalogStorage(butlerLocation)
def getParentSymlinkPath(root)
def readYamlStorage(butlerLocation)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getReadFormatter(cls, objType)
def writeYamlStorage(butlerLocation, obj)
def getLocalFile(self, path)
def read(self, butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)