24 from past.builtins
import basestring
35 from .
import (LogicalLocation, Persistence, Policy, StorageList,
36 StorageInterface, Storage, ButlerLocation,
37 NoRepositroyAtRoot, RepositoryCfg, doImport)
40 from .safeFileIo
import SafeFilename, safeMakeDir
43 __all__ = [
"PosixStorage"]
47 """Defines the interface for a storage location on the local filesystem. 52 URI or path that is used as the storage location. 54 If True a new repository will be created at the root location if it 55 does not exist. If False then a new repository will not be created. 60 If create is False and a repository does not exist at the root 61 specified by uri then NoRepositroyAtRoot is raised. 65 self.
log = Log.getLogger(
"daf.persistence.butler")
67 if self.
root and not os.path.exists(self.
root):
78 return Persistence.getPersistence(persistencePolicy)
81 return 'PosixStorage(root=%s)' % self.
root 84 def _pathFromURI(uri):
85 """Get the path part of the URI""" 86 return urllib.parse.urlparse(uri).path
90 """Get a relative path from a location to a location. 95 A path at which to start. It can be a relative path or an 98 A target location. It can be a relative path or an absolute path. 103 A relative path that describes the path from fromPath to toPath. 105 fromPath = os.path.realpath(fromPath)
106 return os.path.relpath(toPath, fromPath)
110 """Get an absolute path for the path from fromUri to toUri 114 fromPath : the starting location 115 A location at which to start. It can be a relative path or an 117 relativePath : the location relative to fromPath 123 Path that is an absolute path representation of fromPath + 124 relativePath, if one exists. If relativePath is absolute or if 125 fromPath is not related to relativePath then relativePath will be 128 if os.path.isabs(relativePath):
130 fromPath = os.path.realpath(fromPath)
131 return os.path.normpath(os.path.join(fromPath, relativePath))
135 """Get a persisted RepositoryCfg 139 uri : URI or path to a RepositoryCfg 144 A RepositoryCfg instance or None 146 storage = Storage.makeFromURI(uri)
150 locationList=
'repositoryCfg.yaml',
156 return storage.read(location)
160 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
164 locationList=
'repositoryCfg.yaml',
170 storage.write(location, cfg)
174 """Get the mapper class associated with a repository root. 176 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 177 new code and repositories; they should use the Repository parentCfg mechanism. 182 The location of a persisted ReositoryCfg is (new style repos), or 183 the location where a _mapper file is (old style repos). 187 A class object or a class instance, depending on the state of the 188 mapper when the repository was created. 193 cfg = PosixStorage.getRepositoryCfg(root)
199 mapperFile =
"_mapper" 200 while not os.path.exists(os.path.join(basePath, mapperFile)):
202 if os.path.exists(os.path.join(basePath,
"_parent")):
203 basePath = os.path.join(basePath,
"_parent")
208 if mapperFile
is not None:
209 mapperFile = os.path.join(basePath, mapperFile)
212 with open(mapperFile,
"r") as f: 213 mapperName = f.readline().strip() 214 components = mapperName.split(".")
215 if len(components) <= 1:
216 raise RuntimeError(
"Unqualified mapper name %s in %s" %
217 (mapperName, mapperFile))
218 pkg = importlib.import_module(
".".join(components[:-1]))
219 return getattr(pkg, components[-1])
225 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 231 A path to the folder on the local filesystem. 236 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 239 linkpath = os.path.join(root,
'_parent')
240 if os.path.exists(linkpath):
242 return os.readlink(os.path.join(root,
'_parent'))
246 return os.path.join(root,
'_parent')
249 def write(self, butlerLocation, obj):
250 """Writes an object to a location and persistence format specified by 255 butlerLocation : ButlerLocation 256 The location & formatting for the object to be written. 257 obj : object instance 258 The object to be written. 260 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
263 if not writeFormatter:
266 writeFormatter(butlerLocation, obj)
269 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
271 def read(self, butlerLocation):
272 """Read from a butlerLocation. 276 butlerLocation : ButlerLocation 277 The location & formatting for the object(s) to be read. 281 A list of objects as described by the butler location. One item for 282 each location in butlerLocation.getLocations() 285 if not readFormatter:
288 return readFormatter(butlerLocation)
290 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
293 """Implementation of PosixStorage.exists for ButlerLocation objects. 295 storageName = location.getStorageName()
296 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
297 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
298 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
299 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
301 for locationString
in location.getLocations():
302 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
309 """Check if location exists. 313 location : ButlerLocation or string 314 A a string or a ButlerLocation that describes the location of an 315 object in this storage. 320 True if exists, else False. 322 if isinstance(location, ButlerLocation):
329 """Get the full path to the location. 334 return os.path.join(self.
root, location)
338 """Test if a Version 1 Repository exists. 340 Version 1 Repositories only exist in posix storages, do not have a 341 RepositoryCfg file, and contain either a registry.sqlite3 file, a 342 _mapper file, or a _parent link. 347 A path to a folder on the local filesystem. 352 True if the repository at root exists, else False. 354 return os.path.exists(root)
and (
355 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or 356 os.path.exists(os.path.join(root,
"_mapper"))
or 357 os.path.exists(os.path.join(root,
"_parent"))
361 """Copy a file from one location to another on the local filesystem. 366 Path and name of existing file. 368 Path and name of new file. 374 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
377 """Get a handle to a local copy of the file, downloading it to a 382 A path the the file in storage, relative to root. 386 A handle to a local copy of the file. If storage is remote it will be 387 a temporary file. If storage is local it may be the original file or 388 a temporary file. The file name can be gotten via the 'name' property 389 of the returned object. 391 p = os.path.join(self.
root, path)
401 """Search for the given path in this storage instance. 403 If the path contains an HDU indicator (a number in brackets before the 404 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 405 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 406 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 411 A filename (and optionally prefix path) to search for within root. 416 The location that was found, or None if no location was found. 421 def search(root, path, searchParents=False):
422 """Look for the given path in the current root. 424 Also supports searching for the path in Butler v1 repositories by 425 following the Butler v1 _parent symlink 427 If the path contains an HDU indicator (a number in brackets, e.g. 428 'foo.fits[1]', this will be stripped when searching and so 429 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 430 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 435 The path to the root directory. 437 The path to the file within the root directory. 438 searchParents : bool, optional 439 For Butler v1 repositories only, if true and a _parent symlink 440 exists, then the directory at _parent will be searched if the file 441 is not found in the root repository. Will continue searching the 442 parent of the parent until the file is found or no additional 448 The location that was found, or None if no location was found. 454 while len(rootDir) > 1
and rootDir[-1] ==
'/':
455 rootDir = rootDir[:-1]
457 if path.startswith(rootDir +
"/"):
459 path = path[len(rootDir +
'/'):]
461 elif rootDir ==
"/" and path.startswith(
"/"):
466 pathPrefix = os.path.dirname(path)
467 while pathPrefix !=
"" and pathPrefix !=
"/":
468 if os.path.realpath(pathPrefix) == os.path.realpath(root):
470 pathPrefix = os.path.dirname(pathPrefix)
471 if pathPrefix ==
"/":
473 elif pathPrefix !=
"":
474 path = path[len(pathPrefix)+1:]
480 firstBracket = path.find(
"[")
481 if firstBracket != -1:
482 strippedPath = path[:firstBracket]
483 pathStripped = path[firstBracket:]
487 paths = glob.glob(os.path.join(dir, strippedPath))
489 if pathPrefix != rootDir:
490 paths = [p[len(rootDir+
'/'):]
for p
in paths]
491 if pathStripped
is not None:
492 paths = [p + pathStripped
for p
in paths]
495 dir = os.path.join(dir,
"_parent")
496 if not os.path.exists(dir):
503 """Ask if a storage at the location described by uri exists 508 URI to the the root location of the storage 513 True if the storage exists, false if not 515 return os.path.exists(PosixStorage._pathFromURI(uri))
519 """Read an lsst.pex.config.Config from a butlerLocation. 523 butlerLocation : ButlerLocation 524 The location for the object(s) to be read. 528 A list of objects as described by the butler location. One item for 529 each location in butlerLocation.getLocations() 532 for locationString
in butlerLocation.getLocations():
533 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
534 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
535 if not os.path.exists(logLoc.locString()):
536 raise RuntimeError(
"No such config file: " + logLoc.locString())
537 pythonType = butlerLocation.getPythonType()
538 if pythonType
is not None:
539 if isinstance(pythonType, basestring):
540 pythonType = doImport(pythonType)
541 finalItem = pythonType()
542 finalItem.load(logLoc.locString())
543 results.append(finalItem)
548 """Writes an lsst.pex.config.Config object to a location specified by 553 butlerLocation : ButlerLocation 554 The location for the object to be written. 555 obj : object instance 556 The object to be written. 558 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
560 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
561 obj.save(logLoc.locString())
565 """Read objects from a FITS file specified by ButlerLocation. 567 The object is read using class or static method 568 ``readFitsWithOptions(path, options)``, if it exists, else 569 ``readFits(path)``. The ``options`` argument is the data returned by 570 ``butlerLocation.getAdditionalData()``. 574 butlerLocation : ButlerLocation 575 The location for the object(s) to be read. 579 A list of objects as described by the butler location. One item for 580 each location in butlerLocation.getLocations() 582 pythonType = butlerLocation.getPythonType()
583 if pythonType
is not None:
584 if isinstance(pythonType, basestring):
585 pythonType = doImport(pythonType)
586 supportsOptions = hasattr(pythonType,
"readFitsWithOptions")
587 if not supportsOptions:
589 if issubclass(pythonType, (PropertySet, PropertyList)):
590 from lsst.afw.image
import readMetadata
591 reader = readMetadata
593 reader = pythonType.readFits
595 additionalData = butlerLocation.getAdditionalData()
596 for locationString
in butlerLocation.getLocations():
597 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
601 filePath = re.sub(
r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$",
r"\1", logLoc.locString())
602 if not os.path.exists(filePath):
603 raise RuntimeError(
"No such FITS file: " + logLoc.locString())
605 finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
607 fileName = logLoc.locString()
608 mat = re.search(
r"^(.*)\[(\d+)\]$", fileName)
610 if mat
and reader == readMetadata:
611 fileName = mat.group(1)
612 hdu = int(mat.group(2))
614 finalItem = reader(fileName, hdu=hdu)
616 finalItem = reader(fileName)
617 results.append(finalItem)
622 """Writes an object to a FITS file specified by ButlerLocation. 624 The object is written using method 625 ``writeFitsWithOptions(path, options)``, if it exists, else 626 ``writeFits(path)``. The ``options`` argument is the data returned by 627 ``butlerLocation.getAdditionalData()``. 631 butlerLocation : ButlerLocation 632 The location for the object to be written. 633 obj : object instance 634 The object to be written. 636 supportsOptions = hasattr(obj,
"writeFitsWithOptions")
637 additionalData = butlerLocation.getAdditionalData()
638 locations = butlerLocation.getLocations()
639 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
642 obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
644 obj.writeFits(logLoc.locString())
648 """Read a catalog from a Parquet file specified by ButlerLocation. 650 The object returned by this is expected to be a subtype 651 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` 652 that allows for lazy loading of the data. 656 butlerLocation : ButlerLocation 657 The location for the object(s) to be read. 661 A list of objects as described by the butler location. One item for 662 each location in butlerLocation.getLocations() 665 additionalData = butlerLocation.getAdditionalData()
667 for locationString
in butlerLocation.getLocations():
668 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
670 if not os.path.exists(logLoc.locString()):
671 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
673 pythonType = butlerLocation.getPythonType()
674 if pythonType
is not None:
675 if isinstance(pythonType, basestring):
676 pythonType = doImport(pythonType)
678 filename = logLoc.locString()
682 results.append(pythonType(filename=filename))
688 """Writes pandas dataframe to parquet file. 692 butlerLocation : ButlerLocation 693 The location for the object(s) to be read. 694 obj : `lsst.qa.explorer.parquetTable.ParquetTable` 695 Wrapped DataFrame to write. 698 additionalData = butlerLocation.getAdditionalData()
699 locations = butlerLocation.getLocations()
700 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
702 filename = logLoc.locString()
707 """Writes an object to a YAML file specified by ButlerLocation. 711 butlerLocation : ButlerLocation 712 The location for the object to be written. 713 obj : object instance 714 The object to be written. 716 additionalData = butlerLocation.getAdditionalData()
717 locations = butlerLocation.getLocations()
718 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
720 with open(logLoc.locString(),
"w")
as outfile:
721 yaml.dump(obj, outfile)
725 """Read an object from a pickle file specified by ButlerLocation. 729 butlerLocation : ButlerLocation 730 The location for the object(s) to be read. 734 A list of objects as described by the butler location. One item for 735 each location in butlerLocation.getLocations() 739 additionalData = butlerLocation.getAdditionalData()
740 for locationString
in butlerLocation.getLocations():
741 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
743 if not os.path.exists(logLoc.locString()):
744 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
745 with open(logLoc.locString(),
"rb")
as infile:
749 if sys.version_info.major >= 3:
750 finalItem = pickle.load(infile, encoding=
"latin1")
752 finalItem = pickle.load(infile)
753 results.append(finalItem)
758 """Writes an object to a pickle file specified by ButlerLocation. 762 butlerLocation : ButlerLocation 763 The location for the object to be written. 764 obj : object instance 765 The object to be written. 767 additionalData = butlerLocation.getAdditionalData()
768 locations = butlerLocation.getLocations()
769 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
771 with open(logLoc.locString(),
"wb")
as outfile:
772 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
776 """Read a catalog from a FITS table specified by ButlerLocation. 780 butlerLocation : ButlerLocation 781 The location for the object(s) to be read. 785 A list of objects as described by the butler location. One item for 786 each location in butlerLocation.getLocations() 788 pythonType = butlerLocation.getPythonType()
789 if pythonType
is not None:
790 if isinstance(pythonType, basestring):
791 pythonType = doImport(pythonType)
793 additionalData = butlerLocation.getAdditionalData()
794 for locationString
in butlerLocation.getLocations():
795 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
797 if not os.path.exists(logLoc.locString()):
798 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
800 if additionalData.exists(
"hdu"):
801 kwds[
"hdu"] = additionalData.getInt(
"hdu")
802 if additionalData.exists(
"flags"):
803 kwds[
"flags"] = additionalData.getInt(
"flags")
804 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
805 results.append(finalItem)
810 """Writes a catalog to a FITS table specified by ButlerLocation. 814 butlerLocation : ButlerLocation 815 The location for the object to be written. 816 obj : object instance 817 The object to be written. 819 additionalData = butlerLocation.getAdditionalData()
820 locations = butlerLocation.getLocations()
821 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
823 if additionalData.exists(
"flags"):
824 kwds = dict(flags=additionalData.getInt(
"flags"))
827 obj.writeFits(logLoc.locString(), **kwds)
831 """Read from a butlerLocation (always fails for this storage type). 835 butlerLocation : ButlerLocation 836 The location for the object(s) to be read. 840 A list of objects as described by the butler location. One item for 841 each location in butlerLocation.getLocations() 843 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
847 """Writes a matplotlib.figure.Figure to a location, using the template's 848 filename suffix to infer the file format. 852 butlerLocation : ButlerLocation 853 The location for the object to be written. 854 obj : matplotlib.figure.Figure 855 The object to be written. 857 additionalData = butlerLocation.getAdditionalData()
858 locations = butlerLocation.getLocations()
859 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
865 _, ext = os.path.splitext(locations[0])
872 obj.savefig(logLoc.locString(), format=ext)
876 """Read a policy from a PAF file specified by a ButlerLocation. 880 butlerLocation : ButlerLocation 881 The location for the object(s) to be read. 885 A list of objects as described by the butler location. One item for 886 each location in butlerLocation.getLocations() 889 for locationString
in butlerLocation.getLocations():
890 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
891 butlerLocation.getAdditionalData())
892 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
893 results.append(finalItem)
898 """Read an object from a YAML file specified by a butlerLocation. 902 butlerLocation : ButlerLocation 903 The location for the object(s) to be read. 907 A list of objects as described by the butler location. One item for 908 each location in butlerLocation.getLocations() 911 for locationString
in butlerLocation.getLocations():
912 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
913 butlerLocation.getAdditionalData())
914 if not os.path.exists(logLoc.locString()):
915 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
917 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
918 finalItem =
Policy(filePath=logLoc.locString())
920 with open(logLoc.locString(),
"rb")
as infile:
921 finalItem = yaml.load(infile)
922 results.append(finalItem)
927 """Read an object from a boost::serialization file. 931 butlerLocation : ButlerLocation 932 The location for the object(s) to be read. 936 A list of objects as described by the butler location. One item for 937 each location in butlerLocation.getLocations() 940 additionalData = butlerLocation.getAdditionalData()
941 for locationString
in butlerLocation.getLocations():
942 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
943 butlerLocation.getAdditionalData())
945 storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
946 storageList.append(storage)
947 finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
949 results.append(finalItem)
954 """Writes an object via boost::serialization. 958 butlerLocation : ButlerLocation 959 The location for the object to be written. 960 obj : object instance 961 The object to be written. 963 additionalData = butlerLocation.getAdditionalData()
964 location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
969 storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
970 storageList.append(storage)
972 if hasattr(obj,
'__deref__'):
974 PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
976 PosixStorage.getPersistence().persist(obj, storageList, additionalData)
979 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
980 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
981 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
982 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
983 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
984 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
985 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
986 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
988 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
989 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def safeMakeDir(directory)
def writeMatplotlibStorage(butlerLocation, obj)
def getWriteFormatter(cls, objType)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def relativePath(fromPath, toPath)
def writeFitsCatalogStorage(butlerLocation, obj)
def exists(self, location)
def readPafStorage(butlerLocation)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def locationWithRoot(self, location)
def readFitsCatalogStorage(butlerLocation)
def getParentSymlinkPath(root)
def readYamlStorage(butlerLocation)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getReadFormatter(cls, objType)
def writeYamlStorage(butlerLocation, obj)
def getLocalFile(self, path)
def read(self, butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)