34 from .
import (LogicalLocation, Policy,
35 StorageInterface, Storage, ButlerLocation,
36 NoRepositroyAtRoot, RepositoryCfg, doImport)
39 from .safeFileIo
import SafeFilename, safeMakeDir
42 __all__ = [
"PosixStorage"]
46 """Defines the interface for a storage location on the local filesystem. 51 URI or path that is used as the storage location. 53 If True a new repository will be created at the root location if it 54 does not exist. If False then a new repository will not be created. 59 If create is False and a repository does not exist at the root 60 specified by uri then NoRepositroyAtRoot is raised. 64 self.
log = Log.getLogger(
"daf.persistence.butler")
66 if self.
root and not os.path.exists(self.
root):
72 return 'PosixStorage(root=%s)' % self.
root 75 def _pathFromURI(uri):
76 """Get the path part of the URI""" 77 return urllib.parse.urlparse(uri).path
81 """Get a relative path from a location to a location. 86 A path at which to start. It can be a relative path or an 89 A target location. It can be a relative path or an absolute path. 94 A relative path that describes the path from fromPath to toPath. 96 fromPath = os.path.realpath(fromPath)
97 return os.path.relpath(toPath, fromPath)
101 """Get an absolute path for the path from fromUri to toUri 105 fromPath : the starting location 106 A location at which to start. It can be a relative path or an 108 relativePath : the location relative to fromPath 114 Path that is an absolute path representation of fromPath + 115 relativePath, if one exists. If relativePath is absolute or if 116 fromPath is not related to relativePath then relativePath will be 119 if os.path.isabs(relativePath):
121 fromPath = os.path.realpath(fromPath)
122 return os.path.normpath(os.path.join(fromPath, relativePath))
126 """Get a persisted RepositoryCfg 130 uri : URI or path to a RepositoryCfg 135 A RepositoryCfg instance or None 137 storage = Storage.makeFromURI(uri)
141 locationList=
'repositoryCfg.yaml',
147 return storage.read(location)
151 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
155 locationList=
'repositoryCfg.yaml',
161 storage.write(location, cfg)
165 """Get the mapper class associated with a repository root. 167 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by 168 new code and repositories; they should use the Repository parentCfg mechanism. 173 The location of a persisted ReositoryCfg is (new style repos), or 174 the location where a _mapper file is (old style repos). 178 A class object or a class instance, depending on the state of the 179 mapper when the repository was created. 184 cfg = PosixStorage.getRepositoryCfg(root)
190 mapperFile =
"_mapper" 191 while not os.path.exists(os.path.join(basePath, mapperFile)):
193 if os.path.exists(os.path.join(basePath,
"_parent")):
194 basePath = os.path.join(basePath,
"_parent")
199 if mapperFile
is not None:
200 mapperFile = os.path.join(basePath, mapperFile)
203 with open(mapperFile,
"r") as f: 204 mapperName = f.readline().strip() 205 components = mapperName.split(".")
206 if len(components) <= 1:
207 raise RuntimeError(
"Unqualified mapper name %s in %s" %
208 (mapperName, mapperFile))
209 pkg = importlib.import_module(
".".join(components[:-1]))
210 return getattr(pkg, components[-1])
216 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the 222 A path to the folder on the local filesystem. 227 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent 230 linkpath = os.path.join(root,
'_parent')
231 if os.path.exists(linkpath):
233 return os.readlink(os.path.join(root,
'_parent'))
237 return os.path.join(root,
'_parent')
240 def write(self, butlerLocation, obj):
241 """Writes an object to a location and persistence format specified by 246 butlerLocation : ButlerLocation 247 The location & formatting for the object to be written. 248 obj : object instance 249 The object to be written. 251 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
254 if not writeFormatter:
257 writeFormatter(butlerLocation, obj)
260 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
262 def read(self, butlerLocation):
263 """Read from a butlerLocation. 267 butlerLocation : ButlerLocation 268 The location & formatting for the object(s) to be read. 272 A list of objects as described by the butler location. One item for 273 each location in butlerLocation.getLocations() 276 if not readFormatter:
279 return readFormatter(butlerLocation)
281 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
284 """Implementation of PosixStorage.exists for ButlerLocation objects. 286 storageName = location.getStorageName()
287 if storageName
not in (
'FitsStorage',
'PafStorage',
288 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
289 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
290 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
292 for locationString
in location.getLocations():
293 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
300 """Check if location exists. 304 location : ButlerLocation or string 305 A a string or a ButlerLocation that describes the location of an 306 object in this storage. 311 True if exists, else False. 313 if isinstance(location, ButlerLocation):
320 """Get the full path to the location. 325 return os.path.join(self.
root, location)
329 """Test if a Version 1 Repository exists. 331 Version 1 Repositories only exist in posix storages, do not have a 332 RepositoryCfg file, and contain either a registry.sqlite3 file, a 333 _mapper file, or a _parent link. 338 A path to a folder on the local filesystem. 343 True if the repository at root exists, else False. 345 return os.path.exists(root)
and (
346 os.path.exists(os.path.join(root,
"registry.sqlite3"))
347 or os.path.exists(os.path.join(root,
"_mapper"))
348 or os.path.exists(os.path.join(root,
"_parent"))
352 """Copy a file from one location to another on the local filesystem. 357 Path and name of existing file. 359 Path and name of new file. 365 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
368 """Get a handle to a local copy of the file, downloading it to a 373 A path the the file in storage, relative to root. 377 A handle to a local copy of the file. If storage is remote it will be 378 a temporary file. If storage is local it may be the original file or 379 a temporary file. The file name can be gotten via the 'name' property 380 of the returned object. 382 p = os.path.join(self.
root, path)
392 """Search for the given path in this storage instance. 394 If the path contains an HDU indicator (a number in brackets before the 395 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 396 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 397 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 402 A filename (and optionally prefix path) to search for within root. 407 The location that was found, or None if no location was found. 412 def search(root, path, searchParents=False):
413 """Look for the given path in the current root. 415 Also supports searching for the path in Butler v1 repositories by 416 following the Butler v1 _parent symlink 418 If the path contains an HDU indicator (a number in brackets, e.g. 419 'foo.fits[1]', this will be stripped when searching and so 420 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 421 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 426 The path to the root directory. 428 The path to the file within the root directory. 429 searchParents : bool, optional 430 For Butler v1 repositories only, if true and a _parent symlink 431 exists, then the directory at _parent will be searched if the file 432 is not found in the root repository. Will continue searching the 433 parent of the parent until the file is found or no additional 439 The location that was found, or None if no location was found. 445 while len(rootDir) > 1
and rootDir[-1] ==
'/':
446 rootDir = rootDir[:-1]
448 if not path.startswith(
'/'):
451 elif path.startswith(rootDir +
"/"):
453 path = path[len(rootDir +
'/'):]
455 elif rootDir ==
"/" and path.startswith(
"/"):
460 pathPrefix = os.path.dirname(path)
461 while pathPrefix !=
"" and pathPrefix !=
"/":
462 if os.path.realpath(pathPrefix) == os.path.realpath(root):
464 pathPrefix = os.path.dirname(pathPrefix)
465 if pathPrefix ==
"/":
467 elif pathPrefix !=
"":
468 path = path[len(pathPrefix)+1:]
474 firstBracket = path.find(
"[")
475 if firstBracket != -1:
476 strippedPath = path[:firstBracket]
477 pathStripped = path[firstBracket:]
481 paths = glob.glob(os.path.join(dir, strippedPath))
483 if pathPrefix != rootDir:
484 paths = [p[len(rootDir+
'/'):]
for p
in paths]
485 if pathStripped
is not None:
486 paths = [p + pathStripped
for p
in paths]
489 dir = os.path.join(dir,
"_parent")
490 if not os.path.exists(dir):
497 """Ask if a storage at the location described by uri exists 502 URI to the the root location of the storage 507 True if the storage exists, false if not 509 return os.path.exists(PosixStorage._pathFromURI(uri))
513 """Read an lsst.pex.config.Config from a butlerLocation. 517 butlerLocation : ButlerLocation 518 The location for the object(s) to be read. 522 A list of objects as described by the butler location. One item for 523 each location in butlerLocation.getLocations() 526 for locationString
in butlerLocation.getLocations():
527 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
528 logLoc =
LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
529 if not os.path.exists(logLoc.locString()):
530 raise RuntimeError(
"No such config file: " + logLoc.locString())
531 pythonType = butlerLocation.getPythonType()
532 if pythonType
is not None:
533 if isinstance(pythonType, str):
534 pythonType = doImport(pythonType)
535 finalItem = pythonType()
536 finalItem.load(logLoc.locString())
537 results.append(finalItem)
542 """Writes an lsst.pex.config.Config object to a location specified by 547 butlerLocation : ButlerLocation 548 The location for the object to be written. 549 obj : object instance 550 The object to be written. 552 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
554 logLoc =
LogicalLocation(locationString, butlerLocation.getAdditionalData())
555 obj.save(logLoc.locString())
559 """Read objects from a FITS file specified by ButlerLocation. 561 The object is read using class or static method 562 ``readFitsWithOptions(path, options)``, if it exists, else 563 ``readFits(path)``. The ``options`` argument is the data returned by 564 ``butlerLocation.getAdditionalData()``. 568 butlerLocation : ButlerLocation 569 The location for the object(s) to be read. 573 A list of objects as described by the butler location. One item for 574 each location in butlerLocation.getLocations() 576 pythonType = butlerLocation.getPythonType()
577 if pythonType
is not None:
578 if isinstance(pythonType, str):
579 pythonType = doImport(pythonType)
580 supportsOptions = hasattr(pythonType,
"readFitsWithOptions")
581 if not supportsOptions:
583 if issubclass(pythonType, (PropertySet, PropertyList)):
584 from lsst.afw.image
import readMetadata
585 reader = readMetadata
587 reader = pythonType.readFits
589 additionalData = butlerLocation.getAdditionalData()
590 for locationString
in butlerLocation.getLocations():
591 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
595 filePath = re.sub(
r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$",
r"\1", logLoc.locString())
596 if not os.path.exists(filePath):
597 raise RuntimeError(
"No such FITS file: " + logLoc.locString())
599 finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
601 fileName = logLoc.locString()
602 mat = re.search(
r"^(.*)\[(\d+)\]$", fileName)
604 if mat
and reader == readMetadata:
605 fileName = mat.group(1)
606 hdu = int(mat.group(2))
608 finalItem = reader(fileName, hdu=hdu)
610 finalItem = reader(fileName)
611 results.append(finalItem)
616 """Writes an object to a FITS file specified by ButlerLocation. 618 The object is written using method 619 ``writeFitsWithOptions(path, options)``, if it exists, else 620 ``writeFits(path)``. The ``options`` argument is the data returned by 621 ``butlerLocation.getAdditionalData()``. 625 butlerLocation : ButlerLocation 626 The location for the object to be written. 627 obj : object instance 628 The object to be written. 630 supportsOptions = hasattr(obj,
"writeFitsWithOptions")
631 additionalData = butlerLocation.getAdditionalData()
632 locations = butlerLocation.getLocations()
633 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
636 obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
638 obj.writeFits(logLoc.locString())
642 """Read a catalog from a Parquet file specified by ButlerLocation. 644 The object returned by this is expected to be a subtype 645 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` 646 that allows for lazy loading of the data. 650 butlerLocation : ButlerLocation 651 The location for the object(s) to be read. 655 A list of objects as described by the butler location. One item for 656 each location in butlerLocation.getLocations() 659 additionalData = butlerLocation.getAdditionalData()
661 for locationString
in butlerLocation.getLocations():
662 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
664 if not os.path.exists(logLoc.locString()):
665 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
667 pythonType = butlerLocation.getPythonType()
668 if pythonType
is not None:
669 if isinstance(pythonType, str):
670 pythonType = doImport(pythonType)
672 filename = logLoc.locString()
676 results.append(pythonType(filename=filename))
682 """Writes pandas dataframe to parquet file. 686 butlerLocation : ButlerLocation 687 The location for the object(s) to be read. 688 obj : `lsst.qa.explorer.parquetTable.ParquetTable` 689 Wrapped DataFrame to write. 692 additionalData = butlerLocation.getAdditionalData()
693 locations = butlerLocation.getLocations()
694 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
696 filename = logLoc.locString()
701 """Writes an object to a YAML file specified by ButlerLocation. 705 butlerLocation : ButlerLocation 706 The location for the object to be written. 707 obj : object instance 708 The object to be written. 710 additionalData = butlerLocation.getAdditionalData()
711 locations = butlerLocation.getLocations()
712 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
714 with open(logLoc.locString(),
"w")
as outfile:
715 yaml.dump(obj, outfile)
719 """Read an object from a pickle file specified by ButlerLocation. 723 butlerLocation : ButlerLocation 724 The location for the object(s) to be read. 728 A list of objects as described by the butler location. One item for 729 each location in butlerLocation.getLocations() 733 additionalData = butlerLocation.getAdditionalData()
734 for locationString
in butlerLocation.getLocations():
735 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
737 if not os.path.exists(logLoc.locString()):
738 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
739 with open(logLoc.locString(),
"rb")
as infile:
743 if sys.version_info.major >= 3:
744 finalItem = pickle.load(infile, encoding=
"latin1")
746 finalItem = pickle.load(infile)
747 results.append(finalItem)
752 """Writes an object to a pickle file specified by ButlerLocation. 756 butlerLocation : ButlerLocation 757 The location for the object to be written. 758 obj : object instance 759 The object to be written. 761 additionalData = butlerLocation.getAdditionalData()
762 locations = butlerLocation.getLocations()
763 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
765 with open(logLoc.locString(),
"wb")
as outfile:
766 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
770 """Read a catalog from a FITS table specified by ButlerLocation. 774 butlerLocation : ButlerLocation 775 The location for the object(s) to be read. 779 A list of objects as described by the butler location. One item for 780 each location in butlerLocation.getLocations() 782 pythonType = butlerLocation.getPythonType()
783 if pythonType
is not None:
784 if isinstance(pythonType, str):
785 pythonType = doImport(pythonType)
787 additionalData = butlerLocation.getAdditionalData()
788 for locationString
in butlerLocation.getLocations():
789 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
791 if not os.path.exists(logLoc.locString()):
792 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
794 if additionalData.exists(
"hdu"):
795 kwds[
"hdu"] = additionalData.getInt(
"hdu")
796 if additionalData.exists(
"flags"):
797 kwds[
"flags"] = additionalData.getInt(
"flags")
798 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
799 results.append(finalItem)
804 """Writes a catalog to a FITS table specified by ButlerLocation. 808 butlerLocation : ButlerLocation 809 The location for the object to be written. 810 obj : object instance 811 The object to be written. 813 additionalData = butlerLocation.getAdditionalData()
814 locations = butlerLocation.getLocations()
815 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
817 if additionalData.exists(
"flags"):
818 kwds = dict(flags=additionalData.getInt(
"flags"))
821 obj.writeFits(logLoc.locString(), **kwds)
825 """Read from a butlerLocation (always fails for this storage type). 829 butlerLocation : ButlerLocation 830 The location for the object(s) to be read. 834 A list of objects as described by the butler location. One item for 835 each location in butlerLocation.getLocations() 837 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
841 """Writes a matplotlib.figure.Figure to a location, using the template's 842 filename suffix to infer the file format. 846 butlerLocation : ButlerLocation 847 The location for the object to be written. 848 obj : matplotlib.figure.Figure 849 The object to be written. 851 additionalData = butlerLocation.getAdditionalData()
852 locations = butlerLocation.getLocations()
853 with
SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
859 _, ext = os.path.splitext(locations[0])
866 obj.savefig(logLoc.locString(), format=ext)
870 """Read a policy from a PAF file specified by a ButlerLocation. 874 butlerLocation : ButlerLocation 875 The location for the object(s) to be read. 879 A list of objects as described by the butler location. One item for 880 each location in butlerLocation.getLocations() 883 for locationString
in butlerLocation.getLocations():
884 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
885 butlerLocation.getAdditionalData())
886 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
887 results.append(finalItem)
892 """Read an object from a YAML file specified by a butlerLocation. 896 butlerLocation : ButlerLocation 897 The location for the object(s) to be read. 901 A list of objects as described by the butler location. One item for 902 each location in butlerLocation.getLocations() 905 for locationString
in butlerLocation.getLocations():
906 logLoc =
LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
907 butlerLocation.getAdditionalData())
908 if not os.path.exists(logLoc.locString()):
909 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
911 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
912 finalItem =
Policy(filePath=logLoc.locString())
916 loader = yaml.FullLoader
917 except AttributeError:
919 with open(logLoc.locString(),
"rb")
as infile:
920 finalItem = yaml.load(infile, Loader=loader)
921 results.append(finalItem)
925 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
926 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
927 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
928 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
929 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
930 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
931 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
932 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
934 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
935 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def safeMakeDir(directory)
def writeMatplotlibStorage(butlerLocation, obj)
def getWriteFormatter(cls, objType)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readParquetStorage(butlerLocation)
def relativePath(fromPath, toPath)
def writeFitsCatalogStorage(butlerLocation, obj)
def exists(self, location)
def readPafStorage(butlerLocation)
def butlerLocationExists(self, location)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def locationWithRoot(self, location)
def readFitsCatalogStorage(butlerLocation)
def getParentSymlinkPath(root)
def readYamlStorage(butlerLocation)
def absolutePath(fromPath, relativePath)
def instanceSearch(self, path)
def putRepositoryCfg(cfg, loc=None)
def getReadFormatter(cls, objType)
def writeYamlStorage(butlerLocation, obj)
def getLocalFile(self, path)
def read(self, butlerLocation)
def writeFitsStorage(butlerLocation, obj)
def write(self, butlerLocation, obj)
def __init__(self, uri, create)
def getRepositoryCfg(uri)