lsst.daf.persistence  16.0-7-g88875c5
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import re
30 import urllib.parse
31 import glob
32 import shutil
33 import yaml
34 
35 from . import (LogicalLocation, Persistence, Policy, StorageList,
36  StorageInterface, Storage, ButlerLocation,
37  NoRepositroyAtRoot, RepositoryCfg, doImport)
38 from lsst.log import Log
39 import lsst.pex.policy as pexPolicy
40 from .safeFileIo import SafeFilename, safeMakeDir
41 
42 
43 __all__ = ["PosixStorage"]
44 
45 
47  """Defines the interface for a storage location on the local filesystem.
48 
49  Parameters
50  ----------
51  uri : string
52  URI or path that is used as the storage location.
53  create : bool
54  If True a new repository will be created at the root location if it
55  does not exist. If False then a new repository will not be created.
56 
57  Raises
58  ------
59  NoRepositroyAtRoot
60  If create is False and a repository does not exist at the root
61  specified by uri then NoRepositroyAtRoot is raised.
62  """
63 
64  def __init__(self, uri, create):
65  self.log = Log.getLogger("daf.persistence.butler")
66  self.root = self._pathFromURI(uri)
67  if self.root and not os.path.exists(self.root):
68  if not create:
69  raise NoRepositroyAtRoot("No repository at {}".format(uri))
70  safeMakeDir(self.root)
71 
73 
74  @staticmethod
76  # Always use an empty Persistence policy until we can get rid of it
77  persistencePolicy = pexPolicy.Policy()
78  return Persistence.getPersistence(persistencePolicy)
79 
80  def __repr__(self):
81  return 'PosixStorage(root=%s)' % self.root
82 
83  @staticmethod
84  def _pathFromURI(uri):
85  """Get the path part of the URI"""
86  return urllib.parse.urlparse(uri).path
87 
88  @staticmethod
89  def relativePath(fromPath, toPath):
90  """Get a relative path from a location to a location.
91 
92  Parameters
93  ----------
94  fromPath : string
95  A path at which to start. It can be a relative path or an
96  absolute path.
97  toPath : string
98  A target location. It can be a relative path or an absolute path.
99 
100  Returns
101  -------
102  string
103  A relative path that describes the path from fromPath to toPath.
104  """
105  fromPath = os.path.realpath(fromPath)
106  return os.path.relpath(toPath, fromPath)
107 
108  @staticmethod
109  def absolutePath(fromPath, relativePath):
110  """Get an absolute path for the path from fromUri to toUri
111 
112  Parameters
113  ----------
114  fromPath : the starting location
115  A location at which to start. It can be a relative path or an
116  absolute path.
117  relativePath : the location relative to fromPath
118  A relative path.
119 
120  Returns
121  -------
122  string
123  Path that is an absolute path representation of fromPath +
124  relativePath, if one exists. If relativePath is absolute or if
125  fromPath is not related to relativePath then relativePath will be
126  returned.
127  """
128  if os.path.isabs(relativePath):
129  return relativePath
130  fromPath = os.path.realpath(fromPath)
131  return os.path.normpath(os.path.join(fromPath, relativePath))
132 
133  @staticmethod
135  """Get a persisted RepositoryCfg
136 
137  Parameters
138  ----------
139  uri : URI or path to a RepositoryCfg
140  Description
141 
142  Returns
143  -------
144  A RepositoryCfg instance or None
145  """
146  storage = Storage.makeFromURI(uri)
147  location = ButlerLocation(pythonType=RepositoryCfg,
148  cppType=None,
149  storageName=None,
150  locationList='repositoryCfg.yaml',
151  dataId={},
152  mapper=None,
153  storage=storage,
154  usedDataId=None,
155  datasetType=None)
156  return storage.read(location)
157 
158  @staticmethod
159  def putRepositoryCfg(cfg, loc=None):
160  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
161  location = ButlerLocation(pythonType=RepositoryCfg,
162  cppType=None,
163  storageName=None,
164  locationList='repositoryCfg.yaml',
165  dataId={},
166  mapper=None,
167  storage=storage,
168  usedDataId=None,
169  datasetType=None)
170  storage.write(location, cfg)
171 
172  @staticmethod
173  def getMapperClass(root):
174  """Get the mapper class associated with a repository root.
175 
176  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
177  new code and repositories; they should use the Repository parentCfg mechanism.
178 
179  Parameters
180  ----------
181  root : string
182  The location of a persisted ReositoryCfg is (new style repos), or
183  the location where a _mapper file is (old style repos).
184 
185  Returns
186  -------
187  A class object or a class instance, depending on the state of the
188  mapper when the repository was created.
189  """
190  if not (root):
191  return None
192 
193  cfg = PosixStorage.getRepositoryCfg(root)
194  if cfg is not None:
195  return cfg.mapper
196 
197  # Find a "_mapper" file containing the mapper class name
198  basePath = root
199  mapperFile = "_mapper"
200  while not os.path.exists(os.path.join(basePath, mapperFile)):
201  # Break abstraction by following _parent links from CameraMapper
202  if os.path.exists(os.path.join(basePath, "_parent")):
203  basePath = os.path.join(basePath, "_parent")
204  else:
205  mapperFile = None
206  break
207 
208  if mapperFile is not None:
209  mapperFile = os.path.join(basePath, mapperFile)
210 
211  # Read the name of the mapper class and instantiate it
212  with open(mapperFile, "r") as f:
213  mapperName = f.readline().strip()
214  components = mapperName.split(".")
215  if len(components) <= 1:
216  raise RuntimeError("Unqualified mapper name %s in %s" %
217  (mapperName, mapperFile))
218  pkg = importlib.import_module(".".join(components[:-1]))
219  return getattr(pkg, components[-1])
220 
221  return None
222 
223  @staticmethod
225  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
226  symlink.
227 
228  Parameters
229  ----------
230  root : string
231  A path to the folder on the local filesystem.
232 
233  Returns
234  -------
235  string or None
236  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
237  symlink at root.
238  """
239  linkpath = os.path.join(root, '_parent')
240  if os.path.exists(linkpath):
241  try:
242  return os.readlink(os.path.join(root, '_parent'))
243  except OSError:
244  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
245  # location. Allow that; return the path of that folder.
246  return os.path.join(root, '_parent')
247  return None
248 
249  def write(self, butlerLocation, obj):
250  """Writes an object to a location and persistence format specified by
251  ButlerLocation
252 
253  Parameters
254  ----------
255  butlerLocation : ButlerLocation
256  The location & formatting for the object to be written.
257  obj : object instance
258  The object to be written.
259  """
260  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
261 
262  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
263  if not writeFormatter:
264  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
265  if writeFormatter:
266  writeFormatter(butlerLocation, obj)
267  return
268 
269  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
270 
271  def read(self, butlerLocation):
272  """Read from a butlerLocation.
273 
274  Parameters
275  ----------
276  butlerLocation : ButlerLocation
277  The location & formatting for the object(s) to be read.
278 
279  Returns
280  -------
281  A list of objects as described by the butler location. One item for
282  each location in butlerLocation.getLocations()
283  """
284  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
285  if not readFormatter:
286  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
287  if readFormatter:
288  return readFormatter(butlerLocation)
289 
290  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
291 
292  def butlerLocationExists(self, location):
293  """Implementation of PosixStorage.exists for ButlerLocation objects.
294  """
295  storageName = location.getStorageName()
296  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
297  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
298  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
299  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
300  return False
301  for locationString in location.getLocations():
302  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
303  obj = self.instanceSearch(path=logLoc)
304  if obj:
305  return True
306  return False
307 
308  def exists(self, location):
309  """Check if location exists.
310 
311  Parameters
312  ----------
313  location : ButlerLocation or string
314  A a string or a ButlerLocation that describes the location of an
315  object in this storage.
316 
317  Returns
318  -------
319  bool
320  True if exists, else False.
321  """
322  if isinstance(location, ButlerLocation):
323  return self.butlerLocationExists(location)
324 
325  obj = self.instanceSearch(path=location)
326  return bool(obj)
327 
328  def locationWithRoot(self, location):
329  """Get the full path to the location.
330 
331  :param location:
332  :return:
333  """
334  return os.path.join(self.root, location)
335 
336  @staticmethod
337  def v1RepoExists(root):
338  """Test if a Version 1 Repository exists.
339 
340  Version 1 Repositories only exist in posix storages, do not have a
341  RepositoryCfg file, and contain either a registry.sqlite3 file, a
342  _mapper file, or a _parent link.
343 
344  Parameters
345  ----------
346  root : string
347  A path to a folder on the local filesystem.
348 
349  Returns
350  -------
351  bool
352  True if the repository at root exists, else False.
353  """
354  return os.path.exists(root) and (
355  os.path.exists(os.path.join(root, "registry.sqlite3")) or
356  os.path.exists(os.path.join(root, "_mapper")) or
357  os.path.exists(os.path.join(root, "_parent"))
358  )
359 
360  def copyFile(self, fromLocation, toLocation):
361  """Copy a file from one location to another on the local filesystem.
362 
363  Parameters
364  ----------
365  fromLocation : path
366  Path and name of existing file.
367  toLocation : path
368  Path and name of new file.
369 
370  Returns
371  -------
372  None
373  """
374  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
375 
376  def getLocalFile(self, path):
377  """Get a handle to a local copy of the file, downloading it to a
378  temporary if needed.
379 
380  Parameters
381  ----------
382  A path the the file in storage, relative to root.
383 
384  Returns
385  -------
386  A handle to a local copy of the file. If storage is remote it will be
387  a temporary file. If storage is local it may be the original file or
388  a temporary file. The file name can be gotten via the 'name' property
389  of the returned object.
390  """
391  p = os.path.join(self.root, path)
392  try:
393  return open(p)
394  except IOError as e:
395  if e.errno == 2: # 'No such file or directory'
396  return None
397  else:
398  raise e
399 
400  def instanceSearch(self, path):
401  """Search for the given path in this storage instance.
402 
403  If the path contains an HDU indicator (a number in brackets before the
404  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
405  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
406  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
407 
408  Parameters
409  ----------
410  path : string
411  A filename (and optionally prefix path) to search for within root.
412 
413  Returns
414  -------
415  string or None
416  The location that was found, or None if no location was found.
417  """
418  return self.search(self.root, path)
419 
420  @staticmethod
421  def search(root, path, searchParents=False):
422  """Look for the given path in the current root.
423 
424  Also supports searching for the path in Butler v1 repositories by
425  following the Butler v1 _parent symlink
426 
427  If the path contains an HDU indicator (a number in brackets, e.g.
428  'foo.fits[1]', this will be stripped when searching and so
429  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
430  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
431 
432  Parameters
433  ----------
434  root : string
435  The path to the root directory.
436  path : string
437  The path to the file within the root directory.
438  searchParents : bool, optional
439  For Butler v1 repositories only, if true and a _parent symlink
440  exists, then the directory at _parent will be searched if the file
441  is not found in the root repository. Will continue searching the
442  parent of the parent until the file is found or no additional
443  parent exists.
444 
445  Returns
446  -------
447  string or None
448  The location that was found, or None if no location was found.
449  """
450  # Separate path into a root-equivalent prefix (in dir) and the rest
451  # (left in path)
452  rootDir = root
453  # First remove trailing slashes (#2527)
454  while len(rootDir) > 1 and rootDir[-1] == '/':
455  rootDir = rootDir[:-1]
456 
457  if path.startswith(rootDir + "/"):
458  # Common case; we have the same root prefix string
459  path = path[len(rootDir + '/'):]
460  pathPrefix = rootDir
461  elif rootDir == "/" and path.startswith("/"):
462  path = path[1:]
463  pathPrefix = None
464  else:
465  # Search for prefix that is the same as root
466  pathPrefix = os.path.dirname(path)
467  while pathPrefix != "" and pathPrefix != "/":
468  if os.path.realpath(pathPrefix) == os.path.realpath(root):
469  break
470  pathPrefix = os.path.dirname(pathPrefix)
471  if pathPrefix == "/":
472  path = path[1:]
473  elif pathPrefix != "":
474  path = path[len(pathPrefix)+1:]
475 
476  # Now search for the path in the root or its parents
477  # Strip off any cfitsio bracketed extension if present
478  strippedPath = path
479  pathStripped = None
480  firstBracket = path.find("[")
481  if firstBracket != -1:
482  strippedPath = path[:firstBracket]
483  pathStripped = path[firstBracket:]
484 
485  dir = rootDir
486  while True:
487  paths = glob.glob(os.path.join(dir, strippedPath))
488  if len(paths) > 0:
489  if pathPrefix != rootDir:
490  paths = [p[len(rootDir+'/'):] for p in paths]
491  if pathStripped is not None:
492  paths = [p + pathStripped for p in paths]
493  return paths
494  if searchParents:
495  dir = os.path.join(dir, "_parent")
496  if not os.path.exists(dir):
497  return None
498  else:
499  return None
500 
501  @staticmethod
502  def storageExists(uri):
503  """Ask if a storage at the location described by uri exists
504 
505  Parameters
506  ----------
507  root : string
508  URI to the the root location of the storage
509 
510  Returns
511  -------
512  bool
513  True if the storage exists, false if not
514  """
515  return os.path.exists(PosixStorage._pathFromURI(uri))
516 
517 
518 def readConfigStorage(butlerLocation):
519  """Read an lsst.pex.config.Config from a butlerLocation.
520 
521  Parameters
522  ----------
523  butlerLocation : ButlerLocation
524  The location for the object(s) to be read.
525 
526  Returns
527  -------
528  A list of objects as described by the butler location. One item for
529  each location in butlerLocation.getLocations()
530  """
531  results = []
532  for locationString in butlerLocation.getLocations():
533  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
534  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
535  if not os.path.exists(logLoc.locString()):
536  raise RuntimeError("No such config file: " + logLoc.locString())
537  pythonType = butlerLocation.getPythonType()
538  if pythonType is not None:
539  if isinstance(pythonType, basestring):
540  pythonType = doImport(pythonType)
541  finalItem = pythonType()
542  finalItem.load(logLoc.locString())
543  results.append(finalItem)
544  return results
545 
546 
547 def writeConfigStorage(butlerLocation, obj):
548  """Writes an lsst.pex.config.Config object to a location specified by
549  ButlerLocation.
550 
551  Parameters
552  ----------
553  butlerLocation : ButlerLocation
554  The location for the object to be written.
555  obj : object instance
556  The object to be written.
557  """
558  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
559  with SafeFilename(filename) as locationString:
560  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
561  obj.save(logLoc.locString())
562 
563 
564 def readFitsStorage(butlerLocation):
565  """Read objects from a FITS file specified by ButlerLocation.
566 
567  The object is read using class or static method
568  ``readFitsWithOptions(path, options)``, if it exists, else
569  ``readFits(path)``. The ``options`` argument is the data returned by
570  ``butlerLocation.getAdditionalData()``.
571 
572  Parameters
573  ----------
574  butlerLocation : ButlerLocation
575  The location for the object(s) to be read.
576 
577  Returns
578  -------
579  A list of objects as described by the butler location. One item for
580  each location in butlerLocation.getLocations()
581  """
582  pythonType = butlerLocation.getPythonType()
583  if pythonType is not None:
584  if isinstance(pythonType, basestring):
585  pythonType = doImport(pythonType)
586  supportsOptions = hasattr(pythonType, "readFitsWithOptions")
587  if not supportsOptions:
588  from lsst.daf.base import PropertySet, PropertyList
589  if issubclass(pythonType, (PropertySet, PropertyList)):
590  from lsst.afw.image import readMetadata
591  reader = readMetadata
592  else:
593  reader = pythonType.readFits
594  results = []
595  additionalData = butlerLocation.getAdditionalData()
596  for locationString in butlerLocation.getLocations():
597  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
598  logLoc = LogicalLocation(locStringWithRoot, additionalData)
599  # test for existence of file, ignoring trailing [...]
600  # because that can specify the HDU or other information
601  filePath = re.sub(r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$", r"\1", logLoc.locString())
602  if not os.path.exists(filePath):
603  raise RuntimeError("No such FITS file: " + logLoc.locString())
604  if supportsOptions:
605  finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
606  else:
607  finalItem = reader(logLoc.locString())
608  results.append(finalItem)
609  return results
610 
611 
612 def writeFitsStorage(butlerLocation, obj):
613  """Writes an object to a FITS file specified by ButlerLocation.
614 
615  The object is written using method
616  ``writeFitsWithOptions(path, options)``, if it exists, else
617  ``writeFits(path)``. The ``options`` argument is the data returned by
618  ``butlerLocation.getAdditionalData()``.
619 
620  Parameters
621  ----------
622  butlerLocation : ButlerLocation
623  The location for the object to be written.
624  obj : object instance
625  The object to be written.
626  """
627  supportsOptions = hasattr(obj, "writeFitsWithOptions")
628  additionalData = butlerLocation.getAdditionalData()
629  locations = butlerLocation.getLocations()
630  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
631  logLoc = LogicalLocation(locationString, additionalData)
632  if supportsOptions:
633  obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
634  else:
635  obj.writeFits(logLoc.locString())
636 
637 
638 def readParquetStorage(butlerLocation):
639  """Read a catalog from a Parquet file specified by ButlerLocation.
640 
641  The object returned by this is expected to be a subtype
642  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
643  that allows for lazy loading of the data.
644 
645  Parameters
646  ----------
647  butlerLocation : ButlerLocation
648  The location for the object(s) to be read.
649 
650  Returns
651  -------
652  A list of objects as described by the butler location. One item for
653  each location in butlerLocation.getLocations()
654  """
655  results = []
656  additionalData = butlerLocation.getAdditionalData()
657 
658  for locationString in butlerLocation.getLocations():
659  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
660  logLoc = LogicalLocation(locStringWithRoot, additionalData)
661  if not os.path.exists(logLoc.locString()):
662  raise RuntimeError("No such parquet file: " + logLoc.locString())
663 
664  pythonType = butlerLocation.getPythonType()
665  if pythonType is not None:
666  if isinstance(pythonType, basestring):
667  pythonType = doImport(pythonType)
668 
669  filename = logLoc.locString()
670 
671  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
672  # filename should be the first kwarg, but being explicit here.
673  results.append(pythonType(filename=filename))
674 
675  return results
676 
677 
678 def writeParquetStorage(butlerLocation, obj):
679  """Writes pandas dataframe to parquet file.
680 
681  Parameters
682  ----------
683  butlerLocation : ButlerLocation
684  The location for the object(s) to be read.
685  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
686  Wrapped DataFrame to write.
687 
688  """
689  additionalData = butlerLocation.getAdditionalData()
690  locations = butlerLocation.getLocations()
691  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
692  logLoc = LogicalLocation(locationString, additionalData)
693  filename = logLoc.locString()
694  obj.write(filename)
695 
696 
697 def writeYamlStorage(butlerLocation, obj):
698  """Writes an object to a YAML file specified by ButlerLocation.
699 
700  Parameters
701  ----------
702  butlerLocation : ButlerLocation
703  The location for the object to be written.
704  obj : object instance
705  The object to be written.
706  """
707  additionalData = butlerLocation.getAdditionalData()
708  locations = butlerLocation.getLocations()
709  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
710  logLoc = LogicalLocation(locationString, additionalData)
711  with open(logLoc.locString(), "w") as outfile:
712  yaml.dump(obj, outfile)
713 
714 
715 def readPickleStorage(butlerLocation):
716  """Read an object from a pickle file specified by ButlerLocation.
717 
718  Parameters
719  ----------
720  butlerLocation : ButlerLocation
721  The location for the object(s) to be read.
722 
723  Returns
724  -------
725  A list of objects as described by the butler location. One item for
726  each location in butlerLocation.getLocations()
727  """
728  # Create a list of Storages for the item.
729  results = []
730  additionalData = butlerLocation.getAdditionalData()
731  for locationString in butlerLocation.getLocations():
732  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
733  logLoc = LogicalLocation(locStringWithRoot, additionalData)
734  if not os.path.exists(logLoc.locString()):
735  raise RuntimeError("No such pickle file: " + logLoc.locString())
736  with open(logLoc.locString(), "rb") as infile:
737  # py3: We have to specify encoding since some files were written
738  # by python2, and 'latin1' manages that conversion safely. See:
739  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
740  if sys.version_info.major >= 3:
741  finalItem = pickle.load(infile, encoding="latin1")
742  else:
743  finalItem = pickle.load(infile)
744  results.append(finalItem)
745  return results
746 
747 
748 def writePickleStorage(butlerLocation, obj):
749  """Writes an object to a pickle file specified by ButlerLocation.
750 
751  Parameters
752  ----------
753  butlerLocation : ButlerLocation
754  The location for the object to be written.
755  obj : object instance
756  The object to be written.
757  """
758  additionalData = butlerLocation.getAdditionalData()
759  locations = butlerLocation.getLocations()
760  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
761  logLoc = LogicalLocation(locationString, additionalData)
762  with open(logLoc.locString(), "wb") as outfile:
763  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
764 
765 
766 def readFitsCatalogStorage(butlerLocation):
767  """Read a catalog from a FITS table specified by ButlerLocation.
768 
769  Parameters
770  ----------
771  butlerLocation : ButlerLocation
772  The location for the object(s) to be read.
773 
774  Returns
775  -------
776  A list of objects as described by the butler location. One item for
777  each location in butlerLocation.getLocations()
778  """
779  pythonType = butlerLocation.getPythonType()
780  if pythonType is not None:
781  if isinstance(pythonType, basestring):
782  pythonType = doImport(pythonType)
783  results = []
784  additionalData = butlerLocation.getAdditionalData()
785  for locationString in butlerLocation.getLocations():
786  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
787  logLoc = LogicalLocation(locStringWithRoot, additionalData)
788  if not os.path.exists(logLoc.locString()):
789  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
790  kwds = {}
791  if additionalData.exists("hdu"):
792  kwds["hdu"] = additionalData.getInt("hdu")
793  if additionalData.exists("flags"):
794  kwds["flags"] = additionalData.getInt("flags")
795  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
796  results.append(finalItem)
797  return results
798 
799 
800 def writeFitsCatalogStorage(butlerLocation, obj):
801  """Writes a catalog to a FITS table specified by ButlerLocation.
802 
803  Parameters
804  ----------
805  butlerLocation : ButlerLocation
806  The location for the object to be written.
807  obj : object instance
808  The object to be written.
809  """
810  additionalData = butlerLocation.getAdditionalData()
811  locations = butlerLocation.getLocations()
812  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
813  logLoc = LogicalLocation(locationString, additionalData)
814  if additionalData.exists("flags"):
815  kwds = dict(flags=additionalData.getInt("flags"))
816  else:
817  kwds = {}
818  obj.writeFits(logLoc.locString(), **kwds)
819 
820 
821 def readMatplotlibStorage(butlerLocation):
822  """Read from a butlerLocation (always fails for this storage type).
823 
824  Parameters
825  ----------
826  butlerLocation : ButlerLocation
827  The location for the object(s) to be read.
828 
829  Returns
830  -------
831  A list of objects as described by the butler location. One item for
832  each location in butlerLocation.getLocations()
833  """
834  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
835 
836 
837 def writeMatplotlibStorage(butlerLocation, obj):
838  """Writes a matplotlib.figure.Figure to a location, using the template's
839  filename suffix to infer the file format.
840 
841  Parameters
842  ----------
843  butlerLocation : ButlerLocation
844  The location for the object to be written.
845  obj : matplotlib.figure.Figure
846  The object to be written.
847  """
848  additionalData = butlerLocation.getAdditionalData()
849  locations = butlerLocation.getLocations()
850  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
851  logLoc = LogicalLocation(locationString, additionalData)
852  # SafeFilename appends a random suffix, which corrupts the extension
853  # matplotlib uses to guess the file format.
854  # Instead, we extract the extension from the original location
855  # and pass that as the format directly.
856  _, ext = os.path.splitext(locations[0])
857  if ext:
858  ext = ext[1:] # strip off leading '.'
859  else:
860  # If there is no extension, we let matplotlib fall back to its
861  # default.
862  ext = None
863  obj.savefig(logLoc.locString(), format=ext)
864 
865 
866 def readPafStorage(butlerLocation):
867  """Read a policy from a PAF file specified by a ButlerLocation.
868 
869  Parameters
870  ----------
871  butlerLocation : ButlerLocation
872  The location for the object(s) to be read.
873 
874  Returns
875  -------
876  A list of objects as described by the butler location. One item for
877  each location in butlerLocation.getLocations()
878  """
879  results = []
880  for locationString in butlerLocation.getLocations():
881  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
882  butlerLocation.getAdditionalData())
883  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
884  results.append(finalItem)
885  return results
886 
887 
888 def readYamlStorage(butlerLocation):
889  """Read an object from a YAML file specified by a butlerLocation.
890 
891  Parameters
892  ----------
893  butlerLocation : ButlerLocation
894  The location for the object(s) to be read.
895 
896  Returns
897  -------
898  A list of objects as described by the butler location. One item for
899  each location in butlerLocation.getLocations()
900  """
901  results = []
902  for locationString in butlerLocation.getLocations():
903  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
904  butlerLocation.getAdditionalData())
905  if not os.path.exists(logLoc.locString()):
906  raise RuntimeError("No such YAML file: " + logLoc.locString())
907  # Butler Gen2 repository configurations are handled specially
908  if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
909  finalItem = Policy(filePath=logLoc.locString())
910  else:
911  with open(logLoc.locString(), "rb") as infile:
912  finalItem = yaml.load(infile)
913  results.append(finalItem)
914  return results
915 
916 
917 def readBoostStorage(butlerLocation):
918  """Read an object from a boost::serialization file.
919 
920  Parameters
921  ----------
922  butlerLocation : ButlerLocation
923  The location for the object(s) to be read.
924 
925  Returns
926  -------
927  A list of objects as described by the butler location. One item for
928  each location in butlerLocation.getLocations()
929  """
930  results = []
931  additionalData = butlerLocation.getAdditionalData()
932  for locationString in butlerLocation.getLocations():
933  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
934  butlerLocation.getAdditionalData())
935  storageList = StorageList()
936  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
937  storageList.append(storage)
938  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
939  additionalData)
940  results.append(finalItem)
941  return results
942 
943 
944 def writeBoostStorage(butlerLocation, obj):
945  """Writes an object via boost::serialization.
946 
947  Parameters
948  ----------
949  butlerLocation : ButlerLocation
950  The location for the object to be written.
951  obj : object instance
952  The object to be written.
953  """
954  additionalData = butlerLocation.getAdditionalData()
955  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
956  with SafeFilename(location) as locationString:
957  logLoc = LogicalLocation(locationString, additionalData)
958  # Create a list of Storages for the item.
959  storageList = StorageList()
960  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
961  storageList.append(storage)
962  # Persist the item.
963  if hasattr(obj, '__deref__'):
964  # We have a smart pointer, so dereference it.
965  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
966  else:
967  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
968 
969 
970 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
971 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
972 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
973 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
974 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
975 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
976 PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
977 PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
978 
979 Storage.registerStorageClass(scheme='', cls=PosixStorage)
980 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeYamlStorage(butlerLocation, obj)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)