24 from past.builtins
import basestring
33 from .
import (LogicalLocation, Persistence, Policy, StorageList,
34 StorageInterface, Storage, ButlerLocation,
35 NoRepositroyAtRoot, RepositoryCfg)
36 from lsst.log
import Log
37 import lsst.pex.policy
as pexPolicy
38 from .safeFileIo
import SafeFilename, safeMakeDir
42 """Defines the interface for a storage location on the local filesystem.
47 URI or path that is used as the storage location.
49 If True a new repository will be created at the root location if it
50 does not exist. If False then a new repository will not be created.
55 If create is False and a repository does not exist at the root
56 specified by uri then NoRepositroyAtRoot is raised.
60 self.
log = Log.getLogger(
"daf.persistence.butler")
62 if self.
root and not os.path.exists(self.
root):
64 raise NoRepositroyAtRoot(
"No repository at {}".format(uri))
68 persistencePolicy = pexPolicy.Policy()
69 self.
persistence = Persistence.getPersistence(persistencePolicy)
72 return 'PosixStorage(root=%s)' % self.
root
75 def _pathFromURI(uri):
76 """Get the path part of the URI"""
77 return urllib.parse.urlparse(uri).path
81 """Get a relative path from a location to a location.
86 A path at which to start. It can be a relative path or an
89 A target location. It can be a relative path or an absolute path.
94 A relative path that describes the path from fromPath to toPath.
96 fromPath = os.path.realpath(fromPath)
97 return os.path.relpath(toPath, fromPath)
101 """Get an absolute path for the path from fromUri to toUri
105 fromPath : the starting location
106 A location at which to start. It can be a relative path or an
108 relativePath : the location relative to fromPath
114 Path that is an absolute path representation of fromPath +
115 relativePath, if one exists. If relativePath is absolute or if
116 fromPath is not related to relativePath then relativePath will be
119 if os.path.isabs(relativePath):
121 fromPath = os.path.realpath(fromPath)
122 return os.path.normpath(os.path.join(fromPath, relativePath))
126 """Get a persisted RepositoryCfg
130 uri : URI or path to a RepositoryCfg
135 A RepositoryCfg instance or None
137 storage = Storage.makeFromURI(uri)
138 formatter = storage._getFormatter(RepositoryCfg)
139 return formatter.read(ButlerLocation(pythonType=
None,
142 locationList=
'repositoryCfg.yaml',
151 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
152 formatter = storage._getFormatter(type(cfg))
153 formatter.write(cfg, ButlerLocation(pythonType=
None,
156 locationList=
'repositoryCfg.yaml',
165 """Get the mapper class associated with a repository root.
167 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
168 new code and repositories; they should use the Repository parentCfg mechanism.
173 The location of a persisted ReositoryCfg is (new style repos), or
174 the location where a _mapper file is (old style repos).
178 A class object or a class instance, depending on the state of the
179 mapper when the repository was created.
184 cfg = PosixStorage.getRepositoryCfg(root)
190 mapperFile =
"_mapper"
191 while not os.path.exists(os.path.join(basePath, mapperFile)):
193 if os.path.exists(os.path.join(basePath,
"_parent")):
194 basePath = os.path.join(basePath,
"_parent")
199 if mapperFile
is not None:
200 mapperFile = os.path.join(basePath, mapperFile)
203 with open(mapperFile,
"r") as f:
204 mapperName = f.readline().strip()
205 components = mapperName.split(".")
206 if len(components) <= 1:
207 raise RuntimeError(
"Unqualified mapper name %s in %s" %
208 (mapperName, mapperFile))
209 pkg = importlib.import_module(
".".join(components[:-1]))
210 return getattr(pkg, components[-1])
216 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
222 A path to the folder on the local filesystem.
227 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
230 linkpath = os.path.join(root,
'_parent')
231 if os.path.exists(linkpath):
233 return os.readlink(os.path.join(root,
'_parent'))
237 return os.path.join(root,
'_parent')
240 def write(self, butlerLocation, obj):
241 """Writes an object to a location and persistence format specified by
246 butlerLocation : ButlerLocation
247 The location & formatting for the object to be written.
248 obj : object instance
249 The object to be written.
251 self.log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
253 additionalData = butlerLocation.getAdditionalData()
254 storageName = butlerLocation.getStorageName()
255 locations = butlerLocation.getLocations()
257 pythonType = butlerLocation.getPythonType()
258 if pythonType
is not None:
259 if isinstance(pythonType, basestring):
261 pythonTypeTokenList = pythonType.split(
'.')
262 importClassString = pythonTypeTokenList.pop()
263 importClassString = importClassString.strip()
264 importPackage =
".".join(pythonTypeTokenList)
265 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
266 pythonType = getattr(importType, importClassString)
271 if hasattr(pythonType,
'butlerWrite'):
272 pythonType.butlerWrite(obj, butlerLocation=butlerLocation)
278 if storageName ==
"PickleStorage":
279 with open(logLoc.locString(),
"wb")
as outfile:
280 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
283 if storageName ==
"ConfigStorage":
284 obj.save(logLoc.locString())
287 if storageName ==
"FitsCatalogStorage":
288 if additionalData.exists(
"flags"):
289 kwds = dict(flags=additionalData.getInt(
"flags"))
292 obj.writeFits(logLoc.locString(), **kwds)
297 storage = self.persistence.getPersistStorage(storageName, logLoc)
298 storageList.append(storage)
300 if storageName ==
'FitsStorage':
301 self.persistence.persist(obj, storageList, additionalData)
305 if hasattr(obj,
'__deref__'):
307 self.persistence.persist(obj.__deref__(), storageList, additionalData)
309 self.persistence.persist(obj, storageList, additionalData)
311 def read(self, butlerLocation):
312 """Read from a butlerLocation.
316 butlerLocation : ButlerLocation
317 The location & formatting for the object(s) to be read.
321 A list of objects as described by the butler location. One item for
322 each location in butlerLocation.getLocations()
324 additionalData = butlerLocation.getAdditionalData()
326 storageName = butlerLocation.getStorageName()
328 locations = butlerLocation.getLocations()
329 pythonType = butlerLocation.getPythonType()
330 if pythonType
is not None:
331 if isinstance(pythonType, basestring):
333 pythonTypeTokenList = pythonType.split(
'.')
334 importClassString = pythonTypeTokenList.pop()
335 importClassString = importClassString.strip()
336 importPackage =
".".join(pythonTypeTokenList)
337 importType = __import__(importPackage, globals(), locals(), [importClassString], 0)
338 pythonType = getattr(importType, importClassString)
342 if hasattr(pythonType,
'butlerRead'):
343 results = pythonType.butlerRead(butlerLocation=butlerLocation)
346 for locationString
in locations:
347 locationString = os.path.join(self.
root, locationString)
351 if storageName ==
"PafStorage":
352 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
353 elif storageName ==
"YamlStorage":
354 finalItem = Policy(filePath=logLoc.locString())
355 elif storageName ==
"PickleStorage":
356 if not os.path.exists(logLoc.locString()):
357 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
358 with open(logLoc.locString(),
"rb")
as infile:
362 if sys.version_info.major >= 3:
363 finalItem = pickle.load(infile, encoding=
"latin1")
365 finalItem = pickle.load(infile)
366 elif storageName ==
"FitsCatalogStorage":
367 if not os.path.exists(logLoc.locString()):
368 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
370 if additionalData.exists(
"hdu"):
371 kwds[
"hdu"] = additionalData.getInt(
"hdu")
372 if additionalData.exists(
"flags"):
373 kwds[
"flags"] = additionalData.getInt(
"flags")
374 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
375 elif storageName ==
"ConfigStorage":
376 if not os.path.exists(logLoc.locString()):
377 raise RuntimeError(
"No such config file: " + logLoc.locString())
378 finalItem = pythonType()
379 finalItem.load(logLoc.locString())
382 storage = self.persistence.getRetrieveStorage(storageName, logLoc)
383 storageList.append(storage)
384 finalItem = self.persistence.unsafeRetrieve(
385 butlerLocation.getCppType(), storageList, additionalData)
386 results.append(finalItem)
391 """Implementaion of PosixStorage.exists for ButlerLocation objects."""
392 storageName = location.getStorageName()
393 if storageName
not in (
'BoostStorage',
'FitsStorage',
'PafStorage',
394 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage'):
395 self.log.warn(
"butlerLocationExists for non-supported storage %s" % location)
397 for locationString
in location.getLocations():
398 logLoc =
LogicalLocation(locationString, location.getAdditionalData()).locString()
405 """Check if location exists.
409 location : ButlerLocation or string
410 A a string or a ButlerLocation that describes the location of an
411 object in this storage.
416 True if exists, else False.
418 if isinstance(location, ButlerLocation):
425 """Get the full path to the location.
430 return os.path.join(self.
root, location)
434 """Test if a Version 1 Repository exists.
436 Version 1 Repositories only exist in posix storages, do not have a
437 RepositoryCfg file, and contain either a registry.sqlite3 file, a
438 _mapper file, or a _parent link.
443 A path to a folder on the local filesystem.
448 True if the repository at root exists, else False.
450 return os.path.exists(root)
and (
451 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or
452 os.path.exists(os.path.join(root,
"_mapper"))
or
453 os.path.exists(os.path.join(root,
"_parent"))
457 """Copy a file from one location to another on the local filesystem.
462 Path and name of existing file.
464 Path and name of new file.
470 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
473 """Get a handle to a local copy of the file, downloading it to a
478 A path the the file in storage, relative to root.
482 A handle to a local copy of the file. If storage is remote it will be
483 a temporary file. If storage is local it may be the original file or
484 a temporary file. The file name can be gotten via the 'name' property
485 of the returned object.
487 p = os.path.join(self.
root, path)
497 """Search for the given path in this storage instance.
499 If the path contains an HDU indicator (a number in brackets before the
500 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
501 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
502 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
507 A filename (and optionally prefix path) to search for within root.
512 The location that was found, or None if no location was found.
517 def search(root, path, searchParents=False):
518 """Look for the given path in the current root.
520 Also supports searching for the path in Butler v1 repositories by
521 following the Butler v1 _parent symlink
523 If the path contains an HDU indicator (a number in brackets, e.g.
524 'foo.fits[1]', this will be stripped when searching and so
525 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
526 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
531 The path to the root directory.
533 The path to the file within the root directory.
534 searchParents : bool, optional
535 For Butler v1 repositories only, if true and a _parent symlink
536 exists, then the directory at _parent will be searched if the file
537 is not found in the root repository. Will continue searching the
538 parent of the parent until the file is found or no additional
544 The location that was found, or None if no location was found.
550 while len(rootDir) > 1
and rootDir[-1] ==
'/':
551 rootDir = rootDir[:-1]
553 if path.startswith(rootDir +
"/"):
555 path = path[len(rootDir +
'/'):]
557 elif rootDir ==
"/" and path.startswith(
"/"):
562 pathPrefix = os.path.dirname(path)
563 while pathPrefix !=
"" and pathPrefix !=
"/":
564 if os.path.realpath(pathPrefix) == os.path.realpath(root):
566 pathPrefix = os.path.dirname(pathPrefix)
567 if pathPrefix ==
"/":
569 elif pathPrefix !=
"":
570 path = path[len(pathPrefix)+1:]
576 firstBracket = path.find(
"[")
577 if firstBracket != -1:
578 strippedPath = path[:firstBracket]
579 pathStripped = path[firstBracket:]
583 paths = glob.glob(os.path.join(dir, strippedPath))
585 if pathPrefix != rootDir:
586 paths = [p[len(rootDir+
'/'):]
for p
in paths]
587 if pathStripped
is not None:
588 paths = [p + pathStripped
for p
in paths]
591 dir = os.path.join(dir,
"_parent")
592 if not os.path.exists(dir):
599 """Ask if a storage at the location described by uri exists
604 URI to the the root location of the storage
609 True if the storage exists, false if not
611 return os.path.exists(PosixStorage._pathFromURI(uri))
614 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
615 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)
Class for logical location of a persisted Persistable instance.