34 from .
import (LogicalLocation, Policy,
35 StorageInterface, Storage, ButlerLocation,
36 NoRepositroyAtRoot, RepositoryCfg, doImport)
39 from .safeFileIo
import SafeFilename, safeMakeDir
42 __all__ = [
"PosixStorage"]
46 """Defines the interface for a storage location on the local filesystem.
51 URI or path that is used as the storage location.
53 If True a new repository will be created at the root location if it
54 does not exist. If False then a new repository will not be created.
59 If create is False and a repository does not exist at the root
60 specified by uri then NoRepositroyAtRoot is raised.
64 self.
log = Log.getLogger(
"daf.persistence.butler")
66 if self.
root and not os.path.exists(self.
root):
68 raise NoRepositroyAtRoot(
"No repository at {}".format(uri))
72 return 'PosixStorage(root=%s)' % self.
root
75 def _pathFromURI(uri):
76 """Get the path part of the URI"""
77 return urllib.parse.urlparse(uri).path
81 """Get a relative path from a location to a location.
86 A path at which to start. It can be a relative path or an
89 A target location. It can be a relative path or an absolute path.
94 A relative path that describes the path from fromPath to toPath.
96 fromPath = os.path.realpath(fromPath)
97 return os.path.relpath(toPath, fromPath)
101 """Get an absolute path for the path from fromUri to toUri
105 fromPath : the starting location
106 A location at which to start. It can be a relative path or an
108 relativePath : the location relative to fromPath
114 Path that is an absolute path representation of fromPath +
115 relativePath, if one exists. If relativePath is absolute or if
116 fromPath is not related to relativePath then relativePath will be
119 if os.path.isabs(relativePath):
121 fromPath = os.path.realpath(fromPath)
122 return os.path.normpath(os.path.join(fromPath, relativePath))
126 """Get a persisted RepositoryCfg
130 uri : URI or path to a RepositoryCfg
135 A RepositoryCfg instance or None
137 storage = Storage.makeFromURI(uri)
138 location = ButlerLocation(pythonType=RepositoryCfg,
141 locationList=
'repositoryCfg.yaml',
147 return storage.read(location)
151 storage = Storage.makeFromURI(cfg.root
if loc
is None else loc, create=
True)
152 location = ButlerLocation(pythonType=RepositoryCfg,
155 locationList=
'repositoryCfg.yaml',
161 storage.write(location, cfg)
165 """Get the mapper class associated with a repository root.
167 Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
168 new code and repositories; they should use the Repository parentCfg mechanism.
173 The location of a persisted ReositoryCfg is (new style repos), or
174 the location where a _mapper file is (old style repos).
178 A class object or a class instance, depending on the state of the
179 mapper when the repository was created.
184 cfg = PosixStorage.getRepositoryCfg(root)
190 mapperFile =
"_mapper"
191 while not os.path.exists(os.path.join(basePath, mapperFile)):
193 if os.path.exists(os.path.join(basePath,
"_parent")):
194 basePath = os.path.join(basePath,
"_parent")
199 if mapperFile
is not None:
200 mapperFile = os.path.join(basePath, mapperFile)
203 with open(mapperFile,
"r")
as f:
204 mapperName = f.readline().strip()
205 components = mapperName.split(
".")
206 if len(components) <= 1:
207 raise RuntimeError(
"Unqualified mapper name %s in %s" %
208 (mapperName, mapperFile))
209 pkg = importlib.import_module(
".".join(components[:-1]))
210 return getattr(pkg, components[-1])
216 """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
222 A path to the folder on the local filesystem.
227 A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
230 linkpath = os.path.join(root,
'_parent')
231 if os.path.exists(linkpath):
233 return os.readlink(os.path.join(root,
'_parent'))
237 return os.path.join(root,
'_parent')
240 def write(self, butlerLocation, obj):
241 """Writes an object to a location and persistence format specified by
246 butlerLocation : ButlerLocation
247 The location & formatting for the object to be written.
248 obj : object instance
249 The object to be written.
251 self.
log.debug(
"Put location=%s obj=%s", butlerLocation, obj)
253 writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
254 if not writeFormatter:
255 writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
257 writeFormatter(butlerLocation, obj)
260 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
262 def read(self, butlerLocation):
263 """Read from a butlerLocation.
267 butlerLocation : ButlerLocation
268 The location & formatting for the object(s) to be read.
272 A list of objects as described by the butler location. One item for
273 each location in butlerLocation.getLocations()
275 readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
276 if not readFormatter:
277 readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
279 return readFormatter(butlerLocation)
281 raise(RuntimeError(
"No formatter for location:{}".format(butlerLocation)))
284 """Implementation of PosixStorage.exists for ButlerLocation objects.
286 storageName = location.getStorageName()
287 if storageName
not in (
'FitsStorage',
'PafStorage',
288 'PickleStorage',
'ConfigStorage',
'FitsCatalogStorage',
289 'YamlStorage',
'ParquetStorage',
'MatplotlibStorage'):
290 self.
log.warn(
"butlerLocationExists for non-supported storage %s" % location)
292 for locationString
in location.getLocations():
293 logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
300 """Check if location exists.
304 location : ButlerLocation or string
305 A a string or a ButlerLocation that describes the location of an
306 object in this storage.
311 True if exists, else False.
313 if isinstance(location, ButlerLocation):
320 """Get the full path to the location.
325 return os.path.join(self.
root, location)
329 """Test if a Version 1 Repository exists.
331 Version 1 Repositories only exist in posix storages, do not have a
332 RepositoryCfg file, and contain either a registry.sqlite3 file, a
333 _mapper file, or a _parent link.
338 A path to a folder on the local filesystem.
343 True if the repository at root exists, else False.
345 return os.path.exists(root)
and (
346 os.path.exists(os.path.join(root,
"registry.sqlite3"))
or
347 os.path.exists(os.path.join(root,
"_mapper"))
or
348 os.path.exists(os.path.join(root,
"_parent"))
352 """Copy a file from one location to another on the local filesystem.
357 Path and name of existing file.
359 Path and name of new file.
365 shutil.copy(os.path.join(self.
root, fromLocation), os.path.join(self.
root, toLocation))
368 """Get a handle to a local copy of the file, downloading it to a
373 A path the the file in storage, relative to root.
377 A handle to a local copy of the file. If storage is remote it will be
378 a temporary file. If storage is local it may be the original file or
379 a temporary file. The file name can be gotten via the 'name' property
380 of the returned object.
382 p = os.path.join(self.
root, path)
392 """Search for the given path in this storage instance.
394 If the path contains an HDU indicator (a number in brackets before the
395 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
396 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
397 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
402 A filename (and optionally prefix path) to search for within root.
407 The location that was found, or None if no location was found.
412 def search(root, path, searchParents=False):
413 """Look for the given path in the current root.
415 Also supports searching for the path in Butler v1 repositories by
416 following the Butler v1 _parent symlink
418 If the path contains an HDU indicator (a number in brackets, e.g.
419 'foo.fits[1]', this will be stripped when searching and so
420 will match filenames without the HDU indicator, e.g. 'foo.fits'. The
421 path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
426 The path to the root directory.
428 The path to the file within the root directory.
429 searchParents : bool, optional
430 For Butler v1 repositories only, if true and a _parent symlink
431 exists, then the directory at _parent will be searched if the file
432 is not found in the root repository. Will continue searching the
433 parent of the parent until the file is found or no additional
439 The location that was found, or None if no location was found.
445 while len(rootDir) > 1
and rootDir[-1] ==
'/':
446 rootDir = rootDir[:-1]
448 if path.startswith(rootDir +
"/"):
450 path = path[len(rootDir +
'/'):]
452 elif rootDir ==
"/" and path.startswith(
"/"):
457 pathPrefix = os.path.dirname(path)
458 while pathPrefix !=
"" and pathPrefix !=
"/":
459 if os.path.realpath(pathPrefix) == os.path.realpath(root):
461 pathPrefix = os.path.dirname(pathPrefix)
462 if pathPrefix ==
"/":
464 elif pathPrefix !=
"":
465 path = path[len(pathPrefix)+1:]
471 firstBracket = path.find(
"[")
472 if firstBracket != -1:
473 strippedPath = path[:firstBracket]
474 pathStripped = path[firstBracket:]
478 paths = glob.glob(os.path.join(dir, strippedPath))
480 if pathPrefix != rootDir:
481 paths = [p[len(rootDir+
'/'):]
for p
in paths]
482 if pathStripped
is not None:
483 paths = [p + pathStripped
for p
in paths]
486 dir = os.path.join(dir,
"_parent")
487 if not os.path.exists(dir):
494 """Ask if a storage at the location described by uri exists
499 URI to the the root location of the storage
504 True if the storage exists, false if not
506 return os.path.exists(PosixStorage._pathFromURI(uri))
510 """Read an lsst.pex.config.Config from a butlerLocation.
514 butlerLocation : ButlerLocation
515 The location for the object(s) to be read.
519 A list of objects as described by the butler location. One item for
520 each location in butlerLocation.getLocations()
523 for locationString
in butlerLocation.getLocations():
524 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
525 logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
526 if not os.path.exists(logLoc.locString()):
527 raise RuntimeError(
"No such config file: " + logLoc.locString())
528 pythonType = butlerLocation.getPythonType()
529 if pythonType
is not None:
530 if isinstance(pythonType, str):
531 pythonType = doImport(pythonType)
532 finalItem = pythonType()
533 finalItem.load(logLoc.locString())
534 results.append(finalItem)
539 """Writes an lsst.pex.config.Config object to a location specified by
544 butlerLocation : ButlerLocation
545 The location for the object to be written.
546 obj : object instance
547 The object to be written.
549 filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
551 logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
552 obj.save(logLoc.locString())
556 """Read objects from a FITS file specified by ButlerLocation.
558 The object is read using class or static method
559 ``readFitsWithOptions(path, options)``, if it exists, else
560 ``readFits(path)``. The ``options`` argument is the data returned by
561 ``butlerLocation.getAdditionalData()``.
565 butlerLocation : ButlerLocation
566 The location for the object(s) to be read.
570 A list of objects as described by the butler location. One item for
571 each location in butlerLocation.getLocations()
573 pythonType = butlerLocation.getPythonType()
574 if pythonType
is not None:
575 if isinstance(pythonType, str):
576 pythonType = doImport(pythonType)
577 supportsOptions = hasattr(pythonType,
"readFitsWithOptions")
578 if not supportsOptions:
580 if issubclass(pythonType, (PropertySet, PropertyList)):
581 from lsst.afw.image
import readMetadata
582 reader = readMetadata
584 reader = pythonType.readFits
586 additionalData = butlerLocation.getAdditionalData()
587 for locationString
in butlerLocation.getLocations():
588 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
589 logLoc = LogicalLocation(locStringWithRoot, additionalData)
592 filePath = re.sub(
r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$",
r"\1", logLoc.locString())
593 if not os.path.exists(filePath):
594 raise RuntimeError(
"No such FITS file: " + logLoc.locString())
596 finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
598 fileName = logLoc.locString()
599 mat = re.search(
r"^(.*)\[(\d+)\]$", fileName)
601 if mat
and reader == readMetadata:
602 fileName = mat.group(1)
603 hdu = int(mat.group(2))
605 finalItem = reader(fileName, hdu=hdu)
607 finalItem = reader(fileName)
608 results.append(finalItem)
613 """Writes an object to a FITS file specified by ButlerLocation.
615 The object is written using method
616 ``writeFitsWithOptions(path, options)``, if it exists, else
617 ``writeFits(path)``. The ``options`` argument is the data returned by
618 ``butlerLocation.getAdditionalData()``.
622 butlerLocation : ButlerLocation
623 The location for the object to be written.
624 obj : object instance
625 The object to be written.
627 supportsOptions = hasattr(obj,
"writeFitsWithOptions")
628 additionalData = butlerLocation.getAdditionalData()
629 locations = butlerLocation.getLocations()
630 with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
631 logLoc = LogicalLocation(locationString, additionalData)
633 obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
635 obj.writeFits(logLoc.locString())
639 """Read a catalog from a Parquet file specified by ButlerLocation.
641 The object returned by this is expected to be a subtype
642 of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
643 that allows for lazy loading of the data.
647 butlerLocation : ButlerLocation
648 The location for the object(s) to be read.
652 A list of objects as described by the butler location. One item for
653 each location in butlerLocation.getLocations()
656 additionalData = butlerLocation.getAdditionalData()
658 for locationString
in butlerLocation.getLocations():
659 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
660 logLoc = LogicalLocation(locStringWithRoot, additionalData)
661 if not os.path.exists(logLoc.locString()):
662 raise RuntimeError(
"No such parquet file: " + logLoc.locString())
664 pythonType = butlerLocation.getPythonType()
665 if pythonType
is not None:
666 if isinstance(pythonType, str):
667 pythonType = doImport(pythonType)
669 filename = logLoc.locString()
673 results.append(pythonType(filename=filename))
679 """Writes pandas dataframe to parquet file.
683 butlerLocation : ButlerLocation
684 The location for the object(s) to be read.
685 obj : `lsst.qa.explorer.parquetTable.ParquetTable`
686 Wrapped DataFrame to write.
689 additionalData = butlerLocation.getAdditionalData()
690 locations = butlerLocation.getLocations()
691 with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
692 logLoc = LogicalLocation(locationString, additionalData)
693 filename = logLoc.locString()
698 """Writes an object to a YAML file specified by ButlerLocation.
702 butlerLocation : ButlerLocation
703 The location for the object to be written.
704 obj : object instance
705 The object to be written.
707 additionalData = butlerLocation.getAdditionalData()
708 locations = butlerLocation.getLocations()
709 with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
710 logLoc = LogicalLocation(locationString, additionalData)
711 with open(logLoc.locString(),
"w")
as outfile:
712 yaml.dump(obj, outfile)
716 """Read an object from a pickle file specified by ButlerLocation.
720 butlerLocation : ButlerLocation
721 The location for the object(s) to be read.
725 A list of objects as described by the butler location. One item for
726 each location in butlerLocation.getLocations()
730 additionalData = butlerLocation.getAdditionalData()
731 for locationString
in butlerLocation.getLocations():
732 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
733 logLoc = LogicalLocation(locStringWithRoot, additionalData)
734 if not os.path.exists(logLoc.locString()):
735 raise RuntimeError(
"No such pickle file: " + logLoc.locString())
736 with open(logLoc.locString(),
"rb")
as infile:
740 if sys.version_info.major >= 3:
741 finalItem = pickle.load(infile, encoding=
"latin1")
743 finalItem = pickle.load(infile)
744 results.append(finalItem)
749 """Writes an object to a pickle file specified by ButlerLocation.
753 butlerLocation : ButlerLocation
754 The location for the object to be written.
755 obj : object instance
756 The object to be written.
758 additionalData = butlerLocation.getAdditionalData()
759 locations = butlerLocation.getLocations()
760 with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
761 logLoc = LogicalLocation(locationString, additionalData)
762 with open(logLoc.locString(),
"wb")
as outfile:
763 pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
767 """Read a catalog from a FITS table specified by ButlerLocation.
771 butlerLocation : ButlerLocation
772 The location for the object(s) to be read.
776 A list of objects as described by the butler location. One item for
777 each location in butlerLocation.getLocations()
779 pythonType = butlerLocation.getPythonType()
780 if pythonType
is not None:
781 if isinstance(pythonType, str):
782 pythonType = doImport(pythonType)
784 additionalData = butlerLocation.getAdditionalData()
785 for locationString
in butlerLocation.getLocations():
786 locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
787 logLoc = LogicalLocation(locStringWithRoot, additionalData)
788 if not os.path.exists(logLoc.locString()):
789 raise RuntimeError(
"No such FITS catalog file: " + logLoc.locString())
791 if additionalData.exists(
"hdu"):
792 kwds[
"hdu"] = additionalData.getInt(
"hdu")
793 if additionalData.exists(
"flags"):
794 kwds[
"flags"] = additionalData.getInt(
"flags")
795 finalItem = pythonType.readFits(logLoc.locString(), **kwds)
796 results.append(finalItem)
801 """Writes a catalog to a FITS table specified by ButlerLocation.
805 butlerLocation : ButlerLocation
806 The location for the object to be written.
807 obj : object instance
808 The object to be written.
810 additionalData = butlerLocation.getAdditionalData()
811 locations = butlerLocation.getLocations()
812 with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
813 logLoc = LogicalLocation(locationString, additionalData)
814 if additionalData.exists(
"flags"):
815 kwds = dict(flags=additionalData.getInt(
"flags"))
818 obj.writeFits(logLoc.locString(), **kwds)
822 """Read from a butlerLocation (always fails for this storage type).
826 butlerLocation : ButlerLocation
827 The location for the object(s) to be read.
831 A list of objects as described by the butler location. One item for
832 each location in butlerLocation.getLocations()
834 raise NotImplementedError(
"Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
838 """Writes a matplotlib.figure.Figure to a location, using the template's
839 filename suffix to infer the file format.
843 butlerLocation : ButlerLocation
844 The location for the object to be written.
845 obj : matplotlib.figure.Figure
846 The object to be written.
848 additionalData = butlerLocation.getAdditionalData()
849 locations = butlerLocation.getLocations()
850 with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0]))
as locationString:
851 logLoc = LogicalLocation(locationString, additionalData)
856 _, ext = os.path.splitext(locations[0])
863 obj.savefig(logLoc.locString(), format=ext)
867 """Read a policy from a PAF file specified by a ButlerLocation.
871 butlerLocation : ButlerLocation
872 The location for the object(s) to be read.
876 A list of objects as described by the butler location. One item for
877 each location in butlerLocation.getLocations()
880 for locationString
in butlerLocation.getLocations():
881 logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
882 butlerLocation.getAdditionalData())
883 finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
884 results.append(finalItem)
889 """Read an object from a YAML file specified by a butlerLocation.
893 butlerLocation : ButlerLocation
894 The location for the object(s) to be read.
898 A list of objects as described by the butler location. One item for
899 each location in butlerLocation.getLocations()
902 for locationString
in butlerLocation.getLocations():
903 logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
904 butlerLocation.getAdditionalData())
905 if not os.path.exists(logLoc.locString()):
906 raise RuntimeError(
"No such YAML file: " + logLoc.locString())
908 if butlerLocation.pythonType ==
'lsst.daf.persistence.RepositoryCfg':
909 finalItem = Policy(filePath=logLoc.locString())
913 loader = yaml.FullLoader
914 except AttributeError:
916 with open(logLoc.locString(),
"rb")
as infile:
917 finalItem = yaml.load(infile, Loader=loader)
918 results.append(finalItem)
922 PosixStorage.registerFormatters(
"FitsStorage", readFitsStorage, writeFitsStorage)
923 PosixStorage.registerFormatters(
"ParquetStorage", readParquetStorage, writeParquetStorage)
924 PosixStorage.registerFormatters(
"ConfigStorage", readConfigStorage, writeConfigStorage)
925 PosixStorage.registerFormatters(
"PickleStorage", readPickleStorage, writePickleStorage)
926 PosixStorage.registerFormatters(
"FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
927 PosixStorage.registerFormatters(
"MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
928 PosixStorage.registerFormatters(
"PafStorage", readFormatter=readPafStorage)
929 PosixStorage.registerFormatters(
"YamlStorage", readYamlStorage, writeYamlStorage)
931 Storage.registerStorageClass(scheme=
'', cls=PosixStorage)
932 Storage.registerStorageClass(scheme=
'file', cls=PosixStorage)