lsst.daf.persistence  16.0-8-gfd407c0+1
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import re
30 import urllib.parse
31 import glob
32 import shutil
33 import yaml
34 
35 from . import (LogicalLocation, Persistence, Policy, StorageList,
36  StorageInterface, Storage, ButlerLocation,
37  NoRepositroyAtRoot, RepositoryCfg, doImport)
38 from lsst.log import Log
39 import lsst.pex.policy as pexPolicy
40 from .safeFileIo import SafeFilename, safeMakeDir
41 
42 
43 __all__ = ["PosixStorage"]
44 
45 
47  """Defines the interface for a storage location on the local filesystem.
48 
49  Parameters
50  ----------
51  uri : string
52  URI or path that is used as the storage location.
53  create : bool
54  If True a new repository will be created at the root location if it
55  does not exist. If False then a new repository will not be created.
56 
57  Raises
58  ------
59  NoRepositroyAtRoot
60  If create is False and a repository does not exist at the root
61  specified by uri then NoRepositroyAtRoot is raised.
62  """
63 
64  def __init__(self, uri, create):
65  self.log = Log.getLogger("daf.persistence.butler")
66  self.root = self._pathFromURI(uri)
67  if self.root and not os.path.exists(self.root):
68  if not create:
69  raise NoRepositroyAtRoot("No repository at {}".format(uri))
70  safeMakeDir(self.root)
71 
73 
74  @staticmethod
76  # Always use an empty Persistence policy until we can get rid of it
77  persistencePolicy = pexPolicy.Policy()
78  return Persistence.getPersistence(persistencePolicy)
79 
80  def __repr__(self):
81  return 'PosixStorage(root=%s)' % self.root
82 
83  @staticmethod
84  def _pathFromURI(uri):
85  """Get the path part of the URI"""
86  return urllib.parse.urlparse(uri).path
87 
88  @staticmethod
89  def relativePath(fromPath, toPath):
90  """Get a relative path from a location to a location.
91 
92  Parameters
93  ----------
94  fromPath : string
95  A path at which to start. It can be a relative path or an
96  absolute path.
97  toPath : string
98  A target location. It can be a relative path or an absolute path.
99 
100  Returns
101  -------
102  string
103  A relative path that describes the path from fromPath to toPath.
104  """
105  fromPath = os.path.realpath(fromPath)
106  return os.path.relpath(toPath, fromPath)
107 
108  @staticmethod
109  def absolutePath(fromPath, relativePath):
110  """Get an absolute path for the path from fromUri to toUri
111 
112  Parameters
113  ----------
114  fromPath : the starting location
115  A location at which to start. It can be a relative path or an
116  absolute path.
117  relativePath : the location relative to fromPath
118  A relative path.
119 
120  Returns
121  -------
122  string
123  Path that is an absolute path representation of fromPath +
124  relativePath, if one exists. If relativePath is absolute or if
125  fromPath is not related to relativePath then relativePath will be
126  returned.
127  """
128  if os.path.isabs(relativePath):
129  return relativePath
130  fromPath = os.path.realpath(fromPath)
131  return os.path.normpath(os.path.join(fromPath, relativePath))
132 
133  @staticmethod
135  """Get a persisted RepositoryCfg
136 
137  Parameters
138  ----------
139  uri : URI or path to a RepositoryCfg
140  Description
141 
142  Returns
143  -------
144  A RepositoryCfg instance or None
145  """
146  storage = Storage.makeFromURI(uri)
147  location = ButlerLocation(pythonType=RepositoryCfg,
148  cppType=None,
149  storageName=None,
150  locationList='repositoryCfg.yaml',
151  dataId={},
152  mapper=None,
153  storage=storage,
154  usedDataId=None,
155  datasetType=None)
156  return storage.read(location)
157 
158  @staticmethod
159  def putRepositoryCfg(cfg, loc=None):
160  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
161  location = ButlerLocation(pythonType=RepositoryCfg,
162  cppType=None,
163  storageName=None,
164  locationList='repositoryCfg.yaml',
165  dataId={},
166  mapper=None,
167  storage=storage,
168  usedDataId=None,
169  datasetType=None)
170  storage.write(location, cfg)
171 
172  @staticmethod
173  def getMapperClass(root):
174  """Get the mapper class associated with a repository root.
175 
176  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
177  new code and repositories; they should use the Repository parentCfg mechanism.
178 
179  Parameters
180  ----------
181  root : string
182  The location of a persisted ReositoryCfg is (new style repos), or
183  the location where a _mapper file is (old style repos).
184 
185  Returns
186  -------
187  A class object or a class instance, depending on the state of the
188  mapper when the repository was created.
189  """
190  if not (root):
191  return None
192 
193  cfg = PosixStorage.getRepositoryCfg(root)
194  if cfg is not None:
195  return cfg.mapper
196 
197  # Find a "_mapper" file containing the mapper class name
198  basePath = root
199  mapperFile = "_mapper"
200  while not os.path.exists(os.path.join(basePath, mapperFile)):
201  # Break abstraction by following _parent links from CameraMapper
202  if os.path.exists(os.path.join(basePath, "_parent")):
203  basePath = os.path.join(basePath, "_parent")
204  else:
205  mapperFile = None
206  break
207 
208  if mapperFile is not None:
209  mapperFile = os.path.join(basePath, mapperFile)
210 
211  # Read the name of the mapper class and instantiate it
212  with open(mapperFile, "r") as f:
213  mapperName = f.readline().strip()
214  components = mapperName.split(".")
215  if len(components) <= 1:
216  raise RuntimeError("Unqualified mapper name %s in %s" %
217  (mapperName, mapperFile))
218  pkg = importlib.import_module(".".join(components[:-1]))
219  return getattr(pkg, components[-1])
220 
221  return None
222 
223  @staticmethod
225  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
226  symlink.
227 
228  Parameters
229  ----------
230  root : string
231  A path to the folder on the local filesystem.
232 
233  Returns
234  -------
235  string or None
236  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
237  symlink at root.
238  """
239  linkpath = os.path.join(root, '_parent')
240  if os.path.exists(linkpath):
241  try:
242  return os.readlink(os.path.join(root, '_parent'))
243  except OSError:
244  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
245  # location. Allow that; return the path of that folder.
246  return os.path.join(root, '_parent')
247  return None
248 
249  def write(self, butlerLocation, obj):
250  """Writes an object to a location and persistence format specified by
251  ButlerLocation
252 
253  Parameters
254  ----------
255  butlerLocation : ButlerLocation
256  The location & formatting for the object to be written.
257  obj : object instance
258  The object to be written.
259  """
260  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
261 
262  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
263  if not writeFormatter:
264  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
265  if writeFormatter:
266  writeFormatter(butlerLocation, obj)
267  return
268 
269  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
270 
271  def read(self, butlerLocation):
272  """Read from a butlerLocation.
273 
274  Parameters
275  ----------
276  butlerLocation : ButlerLocation
277  The location & formatting for the object(s) to be read.
278 
279  Returns
280  -------
281  A list of objects as described by the butler location. One item for
282  each location in butlerLocation.getLocations()
283  """
284  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
285  if not readFormatter:
286  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
287  if readFormatter:
288  return readFormatter(butlerLocation)
289 
290  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
291 
292  def butlerLocationExists(self, location):
293  """Implementation of PosixStorage.exists for ButlerLocation objects.
294  """
295  storageName = location.getStorageName()
296  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
297  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
298  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
299  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
300  return False
301  for locationString in location.getLocations():
302  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
303  obj = self.instanceSearch(path=logLoc)
304  if obj:
305  return True
306  return False
307 
308  def exists(self, location):
309  """Check if location exists.
310 
311  Parameters
312  ----------
313  location : ButlerLocation or string
314  A a string or a ButlerLocation that describes the location of an
315  object in this storage.
316 
317  Returns
318  -------
319  bool
320  True if exists, else False.
321  """
322  if isinstance(location, ButlerLocation):
323  return self.butlerLocationExists(location)
324 
325  obj = self.instanceSearch(path=location)
326  return bool(obj)
327 
328  def locationWithRoot(self, location):
329  """Get the full path to the location.
330 
331  :param location:
332  :return:
333  """
334  return os.path.join(self.root, location)
335 
336  @staticmethod
337  def v1RepoExists(root):
338  """Test if a Version 1 Repository exists.
339 
340  Version 1 Repositories only exist in posix storages, do not have a
341  RepositoryCfg file, and contain either a registry.sqlite3 file, a
342  _mapper file, or a _parent link.
343 
344  Parameters
345  ----------
346  root : string
347  A path to a folder on the local filesystem.
348 
349  Returns
350  -------
351  bool
352  True if the repository at root exists, else False.
353  """
354  return os.path.exists(root) and (
355  os.path.exists(os.path.join(root, "registry.sqlite3")) or
356  os.path.exists(os.path.join(root, "_mapper")) or
357  os.path.exists(os.path.join(root, "_parent"))
358  )
359 
360  def copyFile(self, fromLocation, toLocation):
361  """Copy a file from one location to another on the local filesystem.
362 
363  Parameters
364  ----------
365  fromLocation : path
366  Path and name of existing file.
367  toLocation : path
368  Path and name of new file.
369 
370  Returns
371  -------
372  None
373  """
374  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
375 
376  def getLocalFile(self, path):
377  """Get a handle to a local copy of the file, downloading it to a
378  temporary if needed.
379 
380  Parameters
381  ----------
382  A path the the file in storage, relative to root.
383 
384  Returns
385  -------
386  A handle to a local copy of the file. If storage is remote it will be
387  a temporary file. If storage is local it may be the original file or
388  a temporary file. The file name can be gotten via the 'name' property
389  of the returned object.
390  """
391  p = os.path.join(self.root, path)
392  try:
393  return open(p)
394  except IOError as e:
395  if e.errno == 2: # 'No such file or directory'
396  return None
397  else:
398  raise e
399 
400  def instanceSearch(self, path):
401  """Search for the given path in this storage instance.
402 
403  If the path contains an HDU indicator (a number in brackets before the
404  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
405  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
406  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
407 
408  Parameters
409  ----------
410  path : string
411  A filename (and optionally prefix path) to search for within root.
412 
413  Returns
414  -------
415  string or None
416  The location that was found, or None if no location was found.
417  """
418  return self.search(self.root, path)
419 
420  @staticmethod
421  def search(root, path, searchParents=False):
422  """Look for the given path in the current root.
423 
424  Also supports searching for the path in Butler v1 repositories by
425  following the Butler v1 _parent symlink
426 
427  If the path contains an HDU indicator (a number in brackets, e.g.
428  'foo.fits[1]', this will be stripped when searching and so
429  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
430  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
431 
432  Parameters
433  ----------
434  root : string
435  The path to the root directory.
436  path : string
437  The path to the file within the root directory.
438  searchParents : bool, optional
439  For Butler v1 repositories only, if true and a _parent symlink
440  exists, then the directory at _parent will be searched if the file
441  is not found in the root repository. Will continue searching the
442  parent of the parent until the file is found or no additional
443  parent exists.
444 
445  Returns
446  -------
447  string or None
448  The location that was found, or None if no location was found.
449  """
450  # Separate path into a root-equivalent prefix (in dir) and the rest
451  # (left in path)
452  rootDir = root
453  # First remove trailing slashes (#2527)
454  while len(rootDir) > 1 and rootDir[-1] == '/':
455  rootDir = rootDir[:-1]
456 
457  if path.startswith(rootDir + "/"):
458  # Common case; we have the same root prefix string
459  path = path[len(rootDir + '/'):]
460  pathPrefix = rootDir
461  elif rootDir == "/" and path.startswith("/"):
462  path = path[1:]
463  pathPrefix = None
464  else:
465  # Search for prefix that is the same as root
466  pathPrefix = os.path.dirname(path)
467  while pathPrefix != "" and pathPrefix != "/":
468  if os.path.realpath(pathPrefix) == os.path.realpath(root):
469  break
470  pathPrefix = os.path.dirname(pathPrefix)
471  if pathPrefix == "/":
472  path = path[1:]
473  elif pathPrefix != "":
474  path = path[len(pathPrefix)+1:]
475 
476  # Now search for the path in the root or its parents
477  # Strip off any cfitsio bracketed extension if present
478  strippedPath = path
479  pathStripped = None
480  firstBracket = path.find("[")
481  if firstBracket != -1:
482  strippedPath = path[:firstBracket]
483  pathStripped = path[firstBracket:]
484 
485  dir = rootDir
486  while True:
487  paths = glob.glob(os.path.join(dir, strippedPath))
488  if len(paths) > 0:
489  if pathPrefix != rootDir:
490  paths = [p[len(rootDir+'/'):] for p in paths]
491  if pathStripped is not None:
492  paths = [p + pathStripped for p in paths]
493  return paths
494  if searchParents:
495  dir = os.path.join(dir, "_parent")
496  if not os.path.exists(dir):
497  return None
498  else:
499  return None
500 
501  @staticmethod
502  def storageExists(uri):
503  """Ask if a storage at the location described by uri exists
504 
505  Parameters
506  ----------
507  root : string
508  URI to the the root location of the storage
509 
510  Returns
511  -------
512  bool
513  True if the storage exists, false if not
514  """
515  return os.path.exists(PosixStorage._pathFromURI(uri))
516 
517 
518 def readConfigStorage(butlerLocation):
519  """Read an lsst.pex.config.Config from a butlerLocation.
520 
521  Parameters
522  ----------
523  butlerLocation : ButlerLocation
524  The location for the object(s) to be read.
525 
526  Returns
527  -------
528  A list of objects as described by the butler location. One item for
529  each location in butlerLocation.getLocations()
530  """
531  results = []
532  for locationString in butlerLocation.getLocations():
533  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
534  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
535  if not os.path.exists(logLoc.locString()):
536  raise RuntimeError("No such config file: " + logLoc.locString())
537  pythonType = butlerLocation.getPythonType()
538  if pythonType is not None:
539  if isinstance(pythonType, basestring):
540  pythonType = doImport(pythonType)
541  finalItem = pythonType()
542  finalItem.load(logLoc.locString())
543  results.append(finalItem)
544  return results
545 
546 
547 def writeConfigStorage(butlerLocation, obj):
548  """Writes an lsst.pex.config.Config object to a location specified by
549  ButlerLocation.
550 
551  Parameters
552  ----------
553  butlerLocation : ButlerLocation
554  The location for the object to be written.
555  obj : object instance
556  The object to be written.
557  """
558  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
559  with SafeFilename(filename) as locationString:
560  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
561  obj.save(logLoc.locString())
562 
563 
564 def readFitsStorage(butlerLocation):
565  """Read objects from a FITS file specified by ButlerLocation.
566 
567  The object is read using class or static method
568  ``readFitsWithOptions(path, options)``, if it exists, else
569  ``readFits(path)``. The ``options`` argument is the data returned by
570  ``butlerLocation.getAdditionalData()``.
571 
572  Parameters
573  ----------
574  butlerLocation : ButlerLocation
575  The location for the object(s) to be read.
576 
577  Returns
578  -------
579  A list of objects as described by the butler location. One item for
580  each location in butlerLocation.getLocations()
581  """
582  pythonType = butlerLocation.getPythonType()
583  if pythonType is not None:
584  if isinstance(pythonType, basestring):
585  pythonType = doImport(pythonType)
586  supportsOptions = hasattr(pythonType, "readFitsWithOptions")
587  if not supportsOptions:
588  from lsst.daf.base import PropertySet, PropertyList
589  if issubclass(pythonType, (PropertySet, PropertyList)):
590  from lsst.afw.image import readMetadata
591  reader = readMetadata
592  else:
593  reader = pythonType.readFits
594  results = []
595  additionalData = butlerLocation.getAdditionalData()
596  for locationString in butlerLocation.getLocations():
597  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
598  logLoc = LogicalLocation(locStringWithRoot, additionalData)
599  # test for existence of file, ignoring trailing [...]
600  # because that can specify the HDU or other information
601  filePath = re.sub(r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$", r"\1", logLoc.locString())
602  if not os.path.exists(filePath):
603  raise RuntimeError("No such FITS file: " + logLoc.locString())
604  if supportsOptions:
605  finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
606  else:
607  fileName = logLoc.locString()
608  mat = re.search(r"^(.*)\[(\d+)\]$", fileName)
609 
610  if mat and reader == readMetadata: # readMetadata() only understands the hdu argument, not [hdu]
611  fileName = mat.group(1)
612  hdu = int(mat.group(2))
613 
614  finalItem = reader(fileName, hdu=hdu)
615  else:
616  finalItem = reader(fileName)
617  results.append(finalItem)
618  return results
619 
620 
621 def writeFitsStorage(butlerLocation, obj):
622  """Writes an object to a FITS file specified by ButlerLocation.
623 
624  The object is written using method
625  ``writeFitsWithOptions(path, options)``, if it exists, else
626  ``writeFits(path)``. The ``options`` argument is the data returned by
627  ``butlerLocation.getAdditionalData()``.
628 
629  Parameters
630  ----------
631  butlerLocation : ButlerLocation
632  The location for the object to be written.
633  obj : object instance
634  The object to be written.
635  """
636  supportsOptions = hasattr(obj, "writeFitsWithOptions")
637  additionalData = butlerLocation.getAdditionalData()
638  locations = butlerLocation.getLocations()
639  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
640  logLoc = LogicalLocation(locationString, additionalData)
641  if supportsOptions:
642  obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
643  else:
644  obj.writeFits(logLoc.locString())
645 
646 
647 def readParquetStorage(butlerLocation):
648  """Read a catalog from a Parquet file specified by ButlerLocation.
649 
650  The object returned by this is expected to be a subtype
651  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
652  that allows for lazy loading of the data.
653 
654  Parameters
655  ----------
656  butlerLocation : ButlerLocation
657  The location for the object(s) to be read.
658 
659  Returns
660  -------
661  A list of objects as described by the butler location. One item for
662  each location in butlerLocation.getLocations()
663  """
664  results = []
665  additionalData = butlerLocation.getAdditionalData()
666 
667  for locationString in butlerLocation.getLocations():
668  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
669  logLoc = LogicalLocation(locStringWithRoot, additionalData)
670  if not os.path.exists(logLoc.locString()):
671  raise RuntimeError("No such parquet file: " + logLoc.locString())
672 
673  pythonType = butlerLocation.getPythonType()
674  if pythonType is not None:
675  if isinstance(pythonType, basestring):
676  pythonType = doImport(pythonType)
677 
678  filename = logLoc.locString()
679 
680  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
681  # filename should be the first kwarg, but being explicit here.
682  results.append(pythonType(filename=filename))
683 
684  return results
685 
686 
687 def writeParquetStorage(butlerLocation, obj):
688  """Writes pandas dataframe to parquet file.
689 
690  Parameters
691  ----------
692  butlerLocation : ButlerLocation
693  The location for the object(s) to be read.
694  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
695  Wrapped DataFrame to write.
696 
697  """
698  additionalData = butlerLocation.getAdditionalData()
699  locations = butlerLocation.getLocations()
700  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
701  logLoc = LogicalLocation(locationString, additionalData)
702  filename = logLoc.locString()
703  obj.write(filename)
704 
705 
706 def writeYamlStorage(butlerLocation, obj):
707  """Writes an object to a YAML file specified by ButlerLocation.
708 
709  Parameters
710  ----------
711  butlerLocation : ButlerLocation
712  The location for the object to be written.
713  obj : object instance
714  The object to be written.
715  """
716  additionalData = butlerLocation.getAdditionalData()
717  locations = butlerLocation.getLocations()
718  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
719  logLoc = LogicalLocation(locationString, additionalData)
720  with open(logLoc.locString(), "w") as outfile:
721  yaml.dump(obj, outfile)
722 
723 
724 def readPickleStorage(butlerLocation):
725  """Read an object from a pickle file specified by ButlerLocation.
726 
727  Parameters
728  ----------
729  butlerLocation : ButlerLocation
730  The location for the object(s) to be read.
731 
732  Returns
733  -------
734  A list of objects as described by the butler location. One item for
735  each location in butlerLocation.getLocations()
736  """
737  # Create a list of Storages for the item.
738  results = []
739  additionalData = butlerLocation.getAdditionalData()
740  for locationString in butlerLocation.getLocations():
741  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
742  logLoc = LogicalLocation(locStringWithRoot, additionalData)
743  if not os.path.exists(logLoc.locString()):
744  raise RuntimeError("No such pickle file: " + logLoc.locString())
745  with open(logLoc.locString(), "rb") as infile:
746  # py3: We have to specify encoding since some files were written
747  # by python2, and 'latin1' manages that conversion safely. See:
748  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
749  if sys.version_info.major >= 3:
750  finalItem = pickle.load(infile, encoding="latin1")
751  else:
752  finalItem = pickle.load(infile)
753  results.append(finalItem)
754  return results
755 
756 
757 def writePickleStorage(butlerLocation, obj):
758  """Writes an object to a pickle file specified by ButlerLocation.
759 
760  Parameters
761  ----------
762  butlerLocation : ButlerLocation
763  The location for the object to be written.
764  obj : object instance
765  The object to be written.
766  """
767  additionalData = butlerLocation.getAdditionalData()
768  locations = butlerLocation.getLocations()
769  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
770  logLoc = LogicalLocation(locationString, additionalData)
771  with open(logLoc.locString(), "wb") as outfile:
772  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
773 
774 
775 def readFitsCatalogStorage(butlerLocation):
776  """Read a catalog from a FITS table specified by ButlerLocation.
777 
778  Parameters
779  ----------
780  butlerLocation : ButlerLocation
781  The location for the object(s) to be read.
782 
783  Returns
784  -------
785  A list of objects as described by the butler location. One item for
786  each location in butlerLocation.getLocations()
787  """
788  pythonType = butlerLocation.getPythonType()
789  if pythonType is not None:
790  if isinstance(pythonType, basestring):
791  pythonType = doImport(pythonType)
792  results = []
793  additionalData = butlerLocation.getAdditionalData()
794  for locationString in butlerLocation.getLocations():
795  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
796  logLoc = LogicalLocation(locStringWithRoot, additionalData)
797  if not os.path.exists(logLoc.locString()):
798  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
799  kwds = {}
800  if additionalData.exists("hdu"):
801  kwds["hdu"] = additionalData.getInt("hdu")
802  if additionalData.exists("flags"):
803  kwds["flags"] = additionalData.getInt("flags")
804  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
805  results.append(finalItem)
806  return results
807 
808 
809 def writeFitsCatalogStorage(butlerLocation, obj):
810  """Writes a catalog to a FITS table specified by ButlerLocation.
811 
812  Parameters
813  ----------
814  butlerLocation : ButlerLocation
815  The location for the object to be written.
816  obj : object instance
817  The object to be written.
818  """
819  additionalData = butlerLocation.getAdditionalData()
820  locations = butlerLocation.getLocations()
821  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
822  logLoc = LogicalLocation(locationString, additionalData)
823  if additionalData.exists("flags"):
824  kwds = dict(flags=additionalData.getInt("flags"))
825  else:
826  kwds = {}
827  obj.writeFits(logLoc.locString(), **kwds)
828 
829 
830 def readMatplotlibStorage(butlerLocation):
831  """Read from a butlerLocation (always fails for this storage type).
832 
833  Parameters
834  ----------
835  butlerLocation : ButlerLocation
836  The location for the object(s) to be read.
837 
838  Returns
839  -------
840  A list of objects as described by the butler location. One item for
841  each location in butlerLocation.getLocations()
842  """
843  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
844 
845 
846 def writeMatplotlibStorage(butlerLocation, obj):
847  """Writes a matplotlib.figure.Figure to a location, using the template's
848  filename suffix to infer the file format.
849 
850  Parameters
851  ----------
852  butlerLocation : ButlerLocation
853  The location for the object to be written.
854  obj : matplotlib.figure.Figure
855  The object to be written.
856  """
857  additionalData = butlerLocation.getAdditionalData()
858  locations = butlerLocation.getLocations()
859  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
860  logLoc = LogicalLocation(locationString, additionalData)
861  # SafeFilename appends a random suffix, which corrupts the extension
862  # matplotlib uses to guess the file format.
863  # Instead, we extract the extension from the original location
864  # and pass that as the format directly.
865  _, ext = os.path.splitext(locations[0])
866  if ext:
867  ext = ext[1:] # strip off leading '.'
868  else:
869  # If there is no extension, we let matplotlib fall back to its
870  # default.
871  ext = None
872  obj.savefig(logLoc.locString(), format=ext)
873 
874 
875 def readPafStorage(butlerLocation):
876  """Read a policy from a PAF file specified by a ButlerLocation.
877 
878  Parameters
879  ----------
880  butlerLocation : ButlerLocation
881  The location for the object(s) to be read.
882 
883  Returns
884  -------
885  A list of objects as described by the butler location. One item for
886  each location in butlerLocation.getLocations()
887  """
888  results = []
889  for locationString in butlerLocation.getLocations():
890  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
891  butlerLocation.getAdditionalData())
892  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
893  results.append(finalItem)
894  return results
895 
896 
897 def readYamlStorage(butlerLocation):
898  """Read an object from a YAML file specified by a butlerLocation.
899 
900  Parameters
901  ----------
902  butlerLocation : ButlerLocation
903  The location for the object(s) to be read.
904 
905  Returns
906  -------
907  A list of objects as described by the butler location. One item for
908  each location in butlerLocation.getLocations()
909  """
910  results = []
911  for locationString in butlerLocation.getLocations():
912  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
913  butlerLocation.getAdditionalData())
914  if not os.path.exists(logLoc.locString()):
915  raise RuntimeError("No such YAML file: " + logLoc.locString())
916  # Butler Gen2 repository configurations are handled specially
917  if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
918  finalItem = Policy(filePath=logLoc.locString())
919  else:
920  with open(logLoc.locString(), "rb") as infile:
921  finalItem = yaml.load(infile)
922  results.append(finalItem)
923  return results
924 
925 
926 def readBoostStorage(butlerLocation):
927  """Read an object from a boost::serialization file.
928 
929  Parameters
930  ----------
931  butlerLocation : ButlerLocation
932  The location for the object(s) to be read.
933 
934  Returns
935  -------
936  A list of objects as described by the butler location. One item for
937  each location in butlerLocation.getLocations()
938  """
939  results = []
940  additionalData = butlerLocation.getAdditionalData()
941  for locationString in butlerLocation.getLocations():
942  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
943  butlerLocation.getAdditionalData())
944  storageList = StorageList()
945  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
946  storageList.append(storage)
947  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
948  additionalData)
949  results.append(finalItem)
950  return results
951 
952 
953 def writeBoostStorage(butlerLocation, obj):
954  """Writes an object via boost::serialization.
955 
956  Parameters
957  ----------
958  butlerLocation : ButlerLocation
959  The location for the object to be written.
960  obj : object instance
961  The object to be written.
962  """
963  additionalData = butlerLocation.getAdditionalData()
964  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
965  with SafeFilename(location) as locationString:
966  logLoc = LogicalLocation(locationString, additionalData)
967  # Create a list of Storages for the item.
968  storageList = StorageList()
969  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
970  storageList.append(storage)
971  # Persist the item.
972  if hasattr(obj, '__deref__'):
973  # We have a smart pointer, so dereference it.
974  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
975  else:
976  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
977 
978 
979 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
980 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
981 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
982 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
983 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
984 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
985 PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
986 PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
987 
988 Storage.registerStorageClass(scheme='', cls=PosixStorage)
989 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def readYamlStorage(butlerLocation)
def writeYamlStorage(butlerLocation, obj)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)