Coverage for python/lsst/daf/persistence/posixStorage.py : 53%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
#!/usr/bin/env python
# # LSST Data Management System # Copyright 2016 LSST Corporation. # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. #
StorageInterface, Storage, ButlerLocation, NoRepositroyAtRoot, RepositoryCfg, doImport)
"""Defines the interface for a storage location on the local filesystem.
Parameters ---------- uri : string URI or path that is used as the storage location. create : bool If True a new repository will be created at the root location if it does not exist. If False then a new repository will not be created.
Raises ------ NoRepositroyAtRoot If create is False and a repository does not exist at the root specified by uri then NoRepositroyAtRoot is raised. """
raise NoRepositroyAtRoot("No repository at {}".format(uri))
def getPersistence(): # Always use an empty Persistence policy until we can get rid of it
return 'PosixStorage(root=%s)' % self.root
def _pathFromURI(uri): """Get the path part of the URI"""
def relativePath(fromPath, toPath): """Get a relative path from a location to a location.
Parameters ---------- fromPath : string A path at which to start. It can be a relative path or an absolute path. toPath : string A target location. It can be a relative path or an absolute path.
Returns ------- string A relative path that describes the path from fromPath to toPath. """
def absolutePath(fromPath, relativePath): """Get an absolute path for the path from fromUri to toUri
Parameters ---------- fromPath : the starting location A location at which to start. It can be a relative path or an absolute path. relativePath : the location relative to fromPath A relative path.
Returns ------- string Path that is an absolute path representation of fromPath + relativePath, if one exists. If relativePath is absolute or if fromPath is not related to relativePath then relativePath will be returned. """
def getRepositoryCfg(uri): """Get a persisted RepositoryCfg
Parameters ---------- uri : URI or path to a RepositoryCfg Description
Returns ------- A RepositoryCfg instance or None """ cppType=None, storageName=None, locationList='repositoryCfg.yaml', dataId={}, mapper=None, storage=storage, usedDataId=None, datasetType=None)
cppType=None, storageName=None, locationList='repositoryCfg.yaml', dataId={}, mapper=None, storage=storage, usedDataId=None, datasetType=None)
def getMapperClass(root): """Get the mapper class associated with a repository root.
Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by new code and repositories; they should use the Repository parentCfg mechanism.
Parameters ---------- root : string The location of a persisted ReositoryCfg is (new style repos), or the location where a _mapper file is (old style repos).
Returns ------- A class object or a class instance, depending on the state of the mapper when the repository was created. """ return None
return cfg.mapper
# Find a "_mapper" file containing the mapper class name # Break abstraction by following _parent links from CameraMapper else: mapperFile = None break
# Read the name of the mapper class and instantiate it raise RuntimeError("Unqualified mapper name %s in %s" % (mapperName, mapperFile))
return None
def getParentSymlinkPath(root): """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the symlink.
Parameters ---------- root : string A path to the folder on the local filesystem.
Returns ------- string or None A path to the parent folder indicated by the _parent symlink, or None if there is no _parent symlink at root. """ # some of the unit tests rely on a folder called _parent instead of a symlink to aother # location. Allow that; return the path of that folder.
"""Writes an object to a location and persistence format specified by ButlerLocation
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object to be written. obj : object instance The object to be written. """
raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """
raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
"""Implementation of PosixStorage.exists for ButlerLocation objects. """ 'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage', 'ParquetStorage', 'MatplotlibStorage'): self.log.warn("butlerLocationExists for non-supported storage %s" % location) return False
"""Check if location exists.
Parameters ---------- location : ButlerLocation or string A a string or a ButlerLocation that describes the location of an object in this storage.
Returns ------- bool True if exists, else False. """
obj = self.instanceSearch(path=location) return bool(obj)
"""Get the full path to the location.
:param location: :return: """ return os.path.join(self.root, location)
def v1RepoExists(root): """Test if a Version 1 Repository exists.
Version 1 Repositories only exist in posix storages, do not have a RepositoryCfg file, and contain either a registry.sqlite3 file, a _mapper file, or a _parent link.
Parameters ---------- root : string A path to a folder on the local filesystem.
Returns ------- bool True if the repository at root exists, else False. """ os.path.exists(os.path.join(root, "registry.sqlite3")) or os.path.exists(os.path.join(root, "_mapper")) or os.path.exists(os.path.join(root, "_parent")) )
"""Copy a file from one location to another on the local filesystem.
Parameters ---------- fromLocation : path Path and name of existing file. toLocation : path Path and name of new file.
Returns ------- None """ shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
"""Get a handle to a local copy of the file, downloading it to a temporary if needed.
Parameters ---------- A path the the file in storage, relative to root.
Returns ------- A handle to a local copy of the file. If storage is remote it will be a temporary file. If storage is local it may be the original file or a temporary file. The file name can be gotten via the 'name' property of the returned object. """ else: raise e
"""Search for the given path in this storage instance.
If the path contains an HDU indicator (a number in brackets before the dot, e.g. 'foo.fits[1]', this will be stripped when searching and so will match filenames without the HDU indicator, e.g. 'foo.fits'. The path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
Parameters ---------- path : string A filename (and optionally prefix path) to search for within root.
Returns ------- string or None The location that was found, or None if no location was found. """
"""Look for the given path in the current root.
Also supports searching for the path in Butler v1 repositories by following the Butler v1 _parent symlink
If the path contains an HDU indicator (a number in brackets, e.g. 'foo.fits[1]', this will be stripped when searching and so will match filenames without the HDU indicator, e.g. 'foo.fits'. The path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
Parameters ---------- root : string The path to the root directory. path : string The path to the file within the root directory. searchParents : bool, optional For Butler v1 repositories only, if true and a _parent symlink exists, then the directory at _parent will be searched if the file is not found in the root repository. Will continue searching the parent of the parent until the file is found or no additional parent exists.
Returns ------- string or None The location that was found, or None if no location was found. """ # Separate path into a root-equivalent prefix (in dir) and the rest # (left in path) # First remove trailing slashes (#2527) rootDir = rootDir[:-1]
# Common case; we have the same root prefix string path = path[1:] pathPrefix = None else: # Search for prefix that is the same as root break path = path[1:] path = path[len(pathPrefix)+1:]
# Now search for the path in the root or its parents # Strip off any cfitsio bracketed extension if present
else:
def storageExists(uri): """Ask if a storage at the location described by uri exists
Parameters ---------- root : string URI to the the root location of the storage
Returns ------- bool True if the storage exists, false if not """ return os.path.exists(PosixStorage._pathFromURI(uri))
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ results = [] for locationString in butlerLocation.getLocations(): locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString) logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData()) if not os.path.exists(logLoc.locString()): raise RuntimeError("No such config file: " + logLoc.locString()) pythonType = butlerLocation.getPythonType() if pythonType is not None: if isinstance(pythonType, basestring): pythonType = doImport(pythonType) finalItem = pythonType() finalItem.load(logLoc.locString()) results.append(finalItem) return results
"""Writes an object to a location and persistence format specified by ButlerLocation
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object to be written. obj : object instance The object to be written. """ filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0]) with SafeFilename(filename) as locationString: logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData()) obj.save(logLoc.locString())
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ results = [] for locationString in butlerLocation.getLocations(): locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString) logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData()) storageList = StorageList() storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc) storageList.append(storage) finalItem = PosixStorage.getPersistence().unsafeRetrieve( butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData()) results.append(finalItem) return results
"""Writes an object to a location and persistence format specified by ButlerLocation
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object to be written. obj : object instance The object to be written. """ location = butlerLocation.getLocations()[0] with SafeFilename(os.path.join(butlerLocation.getStorage().root, location)) as locationString: logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData()) # Create a list of Storages for the item. storageList = StorageList() storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc) storageList.append(storage) persistence = PosixStorage.getPersistence() if hasattr(obj, '__deref__'): # We have a smart pointer, so dereference it. persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData()) else: persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
"""Read from a butlerLocation.
The object returned by this is expected to be a subtype of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile` that allows for lazy loading of the data.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ results = [] additionalData = butlerLocation.getAdditionalData()
for locationString in butlerLocation.getLocations(): locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString) logLoc = LogicalLocation(locStringWithRoot, additionalData) if not os.path.exists(logLoc.locString()): raise RuntimeError("No such parquet file: " + logLoc.locString())
pythonType = butlerLocation.getPythonType() if pythonType is not None: if isinstance(pythonType, basestring): pythonType = doImport(pythonType)
filename = logLoc.locString()
# pythonType will be ParquetTable (or perhaps MultilevelParquetTable) # filename should be the first kwarg, but being explicit here. results.append(pythonType(filename=filename))
return results
"""Writes pandas dataframe to parquet file
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read. obj : `lsst.qa.explorer.parquetTable.ParquetTable` Wrapped DataFrame to write.
""" additionalData = butlerLocation.getAdditionalData() locations = butlerLocation.getLocations() with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString: logLoc = LogicalLocation(locationString, additionalData) filename = logLoc.locString() obj.write(filename)
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ # Create a list of Storages for the item. raise RuntimeError("No such pickle file: " + logLoc.locString()) # py3: We have to specify encoding since some files were written # by python2, and 'latin1' manages that conversion safely. See: # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598 else: finalItem = pickle.load(infile)
"""Writes an object to a location and persistence format specified by ButlerLocation
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object to be written. obj : object instance The object to be written. """
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ pythonType = butlerLocation.getPythonType() if pythonType is not None: if isinstance(pythonType, basestring): pythonType = doImport(pythonType) results = [] additionalData = butlerLocation.getAdditionalData() for locationString in butlerLocation.getLocations(): locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString) logLoc = LogicalLocation(locStringWithRoot, additionalData) if not os.path.exists(logLoc.locString()): raise RuntimeError("No such FITS catalog file: " + logLoc.locString()) kwds = {} if additionalData.exists("hdu"): kwds["hdu"] = additionalData.getInt("hdu") if additionalData.exists("flags"): kwds["flags"] = additionalData.getInt("flags") finalItem = pythonType.readFits(logLoc.locString(), **kwds) results.append(finalItem) return results
"""Writes an object to a location and persistence format specified by ButlerLocation
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object to be written. obj : object instance The object to be written. """ additionalData = butlerLocation.getAdditionalData() locations = butlerLocation.getLocations() with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString: logLoc = LogicalLocation(locationString, additionalData) if additionalData.exists("flags"): kwds = dict(flags=additionalData.getInt("flags")) else: kwds = {} obj.writeFits(logLoc.locString(), **kwds)
"""Read from a butlerLocation (always fails for this storage type).
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
"""Writes a matplotlib.figure.Figure to a location, using the template's filename suffix to infer the file format.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object to be written. obj : matplotlib.figure.Figure The object to be written. """ additionalData = butlerLocation.getAdditionalData() locations = butlerLocation.getLocations() with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString: logLoc = LogicalLocation(locationString, additionalData) # SafeFilename appends a random suffix, which corrupts the extension # matplotlib uses to guess the file format. # Instead, we extract the extension from the original location # and pass that as the format directly. _, ext = os.path.splitext(locations[0]) if ext: ext = ext[1:] # strip off leading '.' else: # If there is no extension, we let matplotlib fall back to its # default. ext = None obj.savefig(logLoc.locString(), format=ext)
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ results = [] for locationString in butlerLocation.getLocations(): logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString), butlerLocation.getAdditionalData()) finalItem = pexPolicy.Policy.createPolicy(logLoc.locString()) results.append(finalItem) return results
"""Read from a butlerLocation.
Parameters ---------- butlerLocation : ButlerLocation The location & formatting for the object(s) to be read.
Returns ------- A list of objects as described by the butler location. One item for each location in butlerLocation.getLocations() """ results = [] for locationString in butlerLocation.getLocations(): logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString), butlerLocation.getAdditionalData()) finalItem = Policy(filePath=logLoc.locString()) results.append(finalItem) return results
results = [] additionalData = butlerLocation.getAdditionalData() for locationString in butlerLocation.getLocations(): logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString), butlerLocation.getAdditionalData()) storageList = StorageList() storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc) storageList.append(storage) finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList, additionalData) results.append(finalItem) return results
additionalData = butlerLocation.getAdditionalData() location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0]) with SafeFilename(location) as locationString: logLoc = LogicalLocation(locationString, additionalData) # Create a list of Storages for the item. storageList = StorageList() storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc) storageList.append(storage) # Persist the item. if hasattr(obj, '__deref__'): # We have a smart pointer, so dereference it. PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData) else: PosixStorage.getPersistence().persist(obj, storageList, additionalData)
|