lsst.daf.persistence  16.0-5-g10498df+1
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import re
30 import urllib.parse
31 import glob
32 import shutil
33 import yaml
34 
35 from . import (LogicalLocation, Persistence, Policy, StorageList,
36  StorageInterface, Storage, ButlerLocation,
37  NoRepositroyAtRoot, RepositoryCfg, doImport)
38 from lsst.log import Log
39 import lsst.pex.policy as pexPolicy
40 from .safeFileIo import SafeFilename, safeMakeDir
41 
42 
43 __all__ = ["PosixStorage"]
44 
45 
47  """Defines the interface for a storage location on the local filesystem.
48 
49  Parameters
50  ----------
51  uri : string
52  URI or path that is used as the storage location.
53  create : bool
54  If True a new repository will be created at the root location if it
55  does not exist. If False then a new repository will not be created.
56 
57  Raises
58  ------
59  NoRepositroyAtRoot
60  If create is False and a repository does not exist at the root
61  specified by uri then NoRepositroyAtRoot is raised.
62  """
63 
64  def __init__(self, uri, create):
65  self.log = Log.getLogger("daf.persistence.butler")
66  self.root = self._pathFromURI(uri)
67  if self.root and not os.path.exists(self.root):
68  if not create:
69  raise NoRepositroyAtRoot("No repository at {}".format(uri))
70  safeMakeDir(self.root)
71 
73 
74  @staticmethod
76  # Always use an empty Persistence policy until we can get rid of it
77  persistencePolicy = pexPolicy.Policy()
78  return Persistence.getPersistence(persistencePolicy)
79 
80  def __repr__(self):
81  return 'PosixStorage(root=%s)' % self.root
82 
83  @staticmethod
84  def _pathFromURI(uri):
85  """Get the path part of the URI"""
86  return urllib.parse.urlparse(uri).path
87 
88  @staticmethod
89  def relativePath(fromPath, toPath):
90  """Get a relative path from a location to a location.
91 
92  Parameters
93  ----------
94  fromPath : string
95  A path at which to start. It can be a relative path or an
96  absolute path.
97  toPath : string
98  A target location. It can be a relative path or an absolute path.
99 
100  Returns
101  -------
102  string
103  A relative path that describes the path from fromPath to toPath.
104  """
105  fromPath = os.path.realpath(fromPath)
106  return os.path.relpath(toPath, fromPath)
107 
108  @staticmethod
109  def absolutePath(fromPath, relativePath):
110  """Get an absolute path for the path from fromUri to toUri
111 
112  Parameters
113  ----------
114  fromPath : the starting location
115  A location at which to start. It can be a relative path or an
116  absolute path.
117  relativePath : the location relative to fromPath
118  A relative path.
119 
120  Returns
121  -------
122  string
123  Path that is an absolute path representation of fromPath +
124  relativePath, if one exists. If relativePath is absolute or if
125  fromPath is not related to relativePath then relativePath will be
126  returned.
127  """
128  if os.path.isabs(relativePath):
129  return relativePath
130  fromPath = os.path.realpath(fromPath)
131  return os.path.normpath(os.path.join(fromPath, relativePath))
132 
133  @staticmethod
135  """Get a persisted RepositoryCfg
136 
137  Parameters
138  ----------
139  uri : URI or path to a RepositoryCfg
140  Description
141 
142  Returns
143  -------
144  A RepositoryCfg instance or None
145  """
146  storage = Storage.makeFromURI(uri)
147  location = ButlerLocation(pythonType=RepositoryCfg,
148  cppType=None,
149  storageName=None,
150  locationList='repositoryCfg.yaml',
151  dataId={},
152  mapper=None,
153  storage=storage,
154  usedDataId=None,
155  datasetType=None)
156  return storage.read(location)
157 
158  @staticmethod
159  def putRepositoryCfg(cfg, loc=None):
160  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
161  location = ButlerLocation(pythonType=RepositoryCfg,
162  cppType=None,
163  storageName=None,
164  locationList='repositoryCfg.yaml',
165  dataId={},
166  mapper=None,
167  storage=storage,
168  usedDataId=None,
169  datasetType=None)
170  storage.write(location, cfg)
171 
172  @staticmethod
173  def getMapperClass(root):
174  """Get the mapper class associated with a repository root.
175 
176  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
177  new code and repositories; they should use the Repository parentCfg mechanism.
178 
179  Parameters
180  ----------
181  root : string
182  The location of a persisted ReositoryCfg is (new style repos), or
183  the location where a _mapper file is (old style repos).
184 
185  Returns
186  -------
187  A class object or a class instance, depending on the state of the
188  mapper when the repository was created.
189  """
190  if not (root):
191  return None
192 
193  cfg = PosixStorage.getRepositoryCfg(root)
194  if cfg is not None:
195  return cfg.mapper
196 
197  # Find a "_mapper" file containing the mapper class name
198  basePath = root
199  mapperFile = "_mapper"
200  while not os.path.exists(os.path.join(basePath, mapperFile)):
201  # Break abstraction by following _parent links from CameraMapper
202  if os.path.exists(os.path.join(basePath, "_parent")):
203  basePath = os.path.join(basePath, "_parent")
204  else:
205  mapperFile = None
206  break
207 
208  if mapperFile is not None:
209  mapperFile = os.path.join(basePath, mapperFile)
210 
211  # Read the name of the mapper class and instantiate it
212  with open(mapperFile, "r") as f:
213  mapperName = f.readline().strip()
214  components = mapperName.split(".")
215  if len(components) <= 1:
216  raise RuntimeError("Unqualified mapper name %s in %s" %
217  (mapperName, mapperFile))
218  pkg = importlib.import_module(".".join(components[:-1]))
219  return getattr(pkg, components[-1])
220 
221  return None
222 
223  @staticmethod
225  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
226  symlink.
227 
228  Parameters
229  ----------
230  root : string
231  A path to the folder on the local filesystem.
232 
233  Returns
234  -------
235  string or None
236  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
237  symlink at root.
238  """
239  linkpath = os.path.join(root, '_parent')
240  if os.path.exists(linkpath):
241  try:
242  return os.readlink(os.path.join(root, '_parent'))
243  except OSError:
244  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
245  # location. Allow that; return the path of that folder.
246  return os.path.join(root, '_parent')
247  return None
248 
249  def write(self, butlerLocation, obj):
250  """Writes an object to a location and persistence format specified by
251  ButlerLocation
252 
253  Parameters
254  ----------
255  butlerLocation : ButlerLocation
256  The location & formatting for the object to be written.
257  obj : object instance
258  The object to be written.
259  """
260  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
261 
262  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
263  if not writeFormatter:
264  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
265  if writeFormatter:
266  writeFormatter(butlerLocation, obj)
267  return
268 
269  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
270 
271  def read(self, butlerLocation):
272  """Read from a butlerLocation.
273 
274  Parameters
275  ----------
276  butlerLocation : ButlerLocation
277  The location & formatting for the object(s) to be read.
278 
279  Returns
280  -------
281  A list of objects as described by the butler location. One item for
282  each location in butlerLocation.getLocations()
283  """
284  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
285  if not readFormatter:
286  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
287  if readFormatter:
288  return readFormatter(butlerLocation)
289 
290  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
291 
292  def butlerLocationExists(self, location):
293  """Implementation of PosixStorage.exists for ButlerLocation objects.
294  """
295  storageName = location.getStorageName()
296  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
297  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
298  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
299  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
300  return False
301  for locationString in location.getLocations():
302  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
303  obj = self.instanceSearch(path=logLoc)
304  if obj:
305  return True
306  return False
307 
308  def exists(self, location):
309  """Check if location exists.
310 
311  Parameters
312  ----------
313  location : ButlerLocation or string
314  A a string or a ButlerLocation that describes the location of an
315  object in this storage.
316 
317  Returns
318  -------
319  bool
320  True if exists, else False.
321  """
322  if isinstance(location, ButlerLocation):
323  return self.butlerLocationExists(location)
324 
325  obj = self.instanceSearch(path=location)
326  return bool(obj)
327 
328  def locationWithRoot(self, location):
329  """Get the full path to the location.
330 
331  :param location:
332  :return:
333  """
334  return os.path.join(self.root, location)
335 
336  @staticmethod
337  def v1RepoExists(root):
338  """Test if a Version 1 Repository exists.
339 
340  Version 1 Repositories only exist in posix storages, do not have a
341  RepositoryCfg file, and contain either a registry.sqlite3 file, a
342  _mapper file, or a _parent link.
343 
344  Parameters
345  ----------
346  root : string
347  A path to a folder on the local filesystem.
348 
349  Returns
350  -------
351  bool
352  True if the repository at root exists, else False.
353  """
354  return os.path.exists(root) and (
355  os.path.exists(os.path.join(root, "registry.sqlite3")) or
356  os.path.exists(os.path.join(root, "_mapper")) or
357  os.path.exists(os.path.join(root, "_parent"))
358  )
359 
360  def copyFile(self, fromLocation, toLocation):
361  """Copy a file from one location to another on the local filesystem.
362 
363  Parameters
364  ----------
365  fromLocation : path
366  Path and name of existing file.
367  toLocation : path
368  Path and name of new file.
369 
370  Returns
371  -------
372  None
373  """
374  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
375 
376  def getLocalFile(self, path):
377  """Get a handle to a local copy of the file, downloading it to a
378  temporary if needed.
379 
380  Parameters
381  ----------
382  A path the the file in storage, relative to root.
383 
384  Returns
385  -------
386  A handle to a local copy of the file. If storage is remote it will be
387  a temporary file. If storage is local it may be the original file or
388  a temporary file. The file name can be gotten via the 'name' property
389  of the returned object.
390  """
391  p = os.path.join(self.root, path)
392  try:
393  return open(p)
394  except IOError as e:
395  if e.errno == 2: # 'No such file or directory'
396  return None
397  else:
398  raise e
399 
400  def instanceSearch(self, path):
401  """Search for the given path in this storage instance.
402 
403  If the path contains an HDU indicator (a number in brackets before the
404  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
405  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
406  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
407 
408  Parameters
409  ----------
410  path : string
411  A filename (and optionally prefix path) to search for within root.
412 
413  Returns
414  -------
415  string or None
416  The location that was found, or None if no location was found.
417  """
418  return self.search(self.root, path)
419 
420  @staticmethod
421  def search(root, path, searchParents=False):
422  """Look for the given path in the current root.
423 
424  Also supports searching for the path in Butler v1 repositories by
425  following the Butler v1 _parent symlink
426 
427  If the path contains an HDU indicator (a number in brackets, e.g.
428  'foo.fits[1]', this will be stripped when searching and so
429  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
430  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
431 
432  Parameters
433  ----------
434  root : string
435  The path to the root directory.
436  path : string
437  The path to the file within the root directory.
438  searchParents : bool, optional
439  For Butler v1 repositories only, if true and a _parent symlink
440  exists, then the directory at _parent will be searched if the file
441  is not found in the root repository. Will continue searching the
442  parent of the parent until the file is found or no additional
443  parent exists.
444 
445  Returns
446  -------
447  string or None
448  The location that was found, or None if no location was found.
449  """
450  # Separate path into a root-equivalent prefix (in dir) and the rest
451  # (left in path)
452  rootDir = root
453  # First remove trailing slashes (#2527)
454  while len(rootDir) > 1 and rootDir[-1] == '/':
455  rootDir = rootDir[:-1]
456 
457  if path.startswith(rootDir + "/"):
458  # Common case; we have the same root prefix string
459  path = path[len(rootDir + '/'):]
460  pathPrefix = rootDir
461  elif rootDir == "/" and path.startswith("/"):
462  path = path[1:]
463  pathPrefix = None
464  else:
465  # Search for prefix that is the same as root
466  pathPrefix = os.path.dirname(path)
467  while pathPrefix != "" and pathPrefix != "/":
468  if os.path.realpath(pathPrefix) == os.path.realpath(root):
469  break
470  pathPrefix = os.path.dirname(pathPrefix)
471  if pathPrefix == "/":
472  path = path[1:]
473  elif pathPrefix != "":
474  path = path[len(pathPrefix)+1:]
475 
476  # Now search for the path in the root or its parents
477  # Strip off any cfitsio bracketed extension if present
478  strippedPath = path
479  pathStripped = None
480  firstBracket = path.find("[")
481  if firstBracket != -1:
482  strippedPath = path[:firstBracket]
483  pathStripped = path[firstBracket:]
484 
485  dir = rootDir
486  while True:
487  paths = glob.glob(os.path.join(dir, strippedPath))
488  if len(paths) > 0:
489  if pathPrefix != rootDir:
490  paths = [p[len(rootDir+'/'):] for p in paths]
491  if pathStripped is not None:
492  paths = [p + pathStripped for p in paths]
493  return paths
494  if searchParents:
495  dir = os.path.join(dir, "_parent")
496  if not os.path.exists(dir):
497  return None
498  else:
499  return None
500 
501  @staticmethod
502  def storageExists(uri):
503  """Ask if a storage at the location described by uri exists
504 
505  Parameters
506  ----------
507  root : string
508  URI to the the root location of the storage
509 
510  Returns
511  -------
512  bool
513  True if the storage exists, false if not
514  """
515  return os.path.exists(PosixStorage._pathFromURI(uri))
516 
517 
518 def readConfigStorage(butlerLocation):
519  """Read an lsst.pex.config.Config from a butlerLocation.
520 
521  Parameters
522  ----------
523  butlerLocation : ButlerLocation
524  The location for the object(s) to be read.
525 
526  Returns
527  -------
528  A list of objects as described by the butler location. One item for
529  each location in butlerLocation.getLocations()
530  """
531  results = []
532  for locationString in butlerLocation.getLocations():
533  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
534  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
535  if not os.path.exists(logLoc.locString()):
536  raise RuntimeError("No such config file: " + logLoc.locString())
537  pythonType = butlerLocation.getPythonType()
538  if pythonType is not None:
539  if isinstance(pythonType, basestring):
540  pythonType = doImport(pythonType)
541  finalItem = pythonType()
542  finalItem.load(logLoc.locString())
543  results.append(finalItem)
544  return results
545 
546 
547 def writeConfigStorage(butlerLocation, obj):
548  """Writes an lsst.pex.config.Config object to a location specified by
549  ButlerLocation.
550 
551  Parameters
552  ----------
553  butlerLocation : ButlerLocation
554  The location for the object to be written.
555  obj : object instance
556  The object to be written.
557  """
558  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
559  with SafeFilename(filename) as locationString:
560  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
561  obj.save(logLoc.locString())
562 
563 
564 def readFitsStorage(butlerLocation):
565  """Read objects from a FITS file specified by ButlerLocation.
566 
567  The object is read using class or static method
568  ``readFitsWithOptions(path, options)``, if it exists, else
569  ``readFits(path)``. The ``options`` argument is the data returned by
570  ``butlerLocation.getAdditionalData()``.
571 
572  Parameters
573  ----------
574  butlerLocation : ButlerLocation
575  The location for the object(s) to be read.
576 
577  Returns
578  -------
579  A list of objects as described by the butler location. One item for
580  each location in butlerLocation.getLocations()
581  """
582  pythonType = butlerLocation.getPythonType()
583  if pythonType is not None:
584  if isinstance(pythonType, basestring):
585  pythonType = doImport(pythonType)
586  supportsOptions = hasattr(pythonType, "readFitsWithOptions")
587  results = []
588  additionalData = butlerLocation.getAdditionalData()
589  for locationString in butlerLocation.getLocations():
590  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
591  logLoc = LogicalLocation(locStringWithRoot, additionalData)
592  # test for existence of file, ignoring trailing [...]
593  # because that can specify the HDU or other information
594  filePath = re.sub(r"(\.fits(.[a-zA-Z0-9]+)?)(\[.+\])$", r"\1", logLoc.locString())
595  if not os.path.exists(filePath):
596  raise RuntimeError("No such FITS file: " + logLoc.locString())
597  if supportsOptions:
598  finalItem = pythonType.readFitsWithOptions(logLoc.locString(), options=additionalData)
599  else:
600  finalItem = pythonType.readFits(logLoc.locString())
601  results.append(finalItem)
602  return results
603 
604 
605 def writeFitsStorage(butlerLocation, obj):
606  """Writes an object to a FITS file specified by ButlerLocation.
607 
608  The object is written using method
609  ``writeFitsWithOptions(path, options)``, if it exists, else
610  ``writeFits(path)``. The ``options`` argument is the data returned by
611  ``butlerLocation.getAdditionalData()``.
612 
613  Parameters
614  ----------
615  butlerLocation : ButlerLocation
616  The location for the object to be written.
617  obj : object instance
618  The object to be written.
619  """
620  supportsOptions = hasattr(obj, "writeFitsWithOptions")
621  additionalData = butlerLocation.getAdditionalData()
622  locations = butlerLocation.getLocations()
623  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
624  logLoc = LogicalLocation(locationString, additionalData)
625  if supportsOptions:
626  obj.writeFitsWithOptions(logLoc.locString(), options=additionalData)
627  else:
628  obj.writeFits(logLoc.locString())
629 
630 
631 def readParquetStorage(butlerLocation):
632  """Read a catalog from a Parquet file specified by ButlerLocation.
633 
634  The object returned by this is expected to be a subtype
635  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
636  that allows for lazy loading of the data.
637 
638  Parameters
639  ----------
640  butlerLocation : ButlerLocation
641  The location for the object(s) to be read.
642 
643  Returns
644  -------
645  A list of objects as described by the butler location. One item for
646  each location in butlerLocation.getLocations()
647  """
648  results = []
649  additionalData = butlerLocation.getAdditionalData()
650 
651  for locationString in butlerLocation.getLocations():
652  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
653  logLoc = LogicalLocation(locStringWithRoot, additionalData)
654  if not os.path.exists(logLoc.locString()):
655  raise RuntimeError("No such parquet file: " + logLoc.locString())
656 
657  pythonType = butlerLocation.getPythonType()
658  if pythonType is not None:
659  if isinstance(pythonType, basestring):
660  pythonType = doImport(pythonType)
661 
662  filename = logLoc.locString()
663 
664  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
665  # filename should be the first kwarg, but being explicit here.
666  results.append(pythonType(filename=filename))
667 
668  return results
669 
670 
671 def writeParquetStorage(butlerLocation, obj):
672  """Writes pandas dataframe to parquet file.
673 
674  Parameters
675  ----------
676  butlerLocation : ButlerLocation
677  The location for the object(s) to be read.
678  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
679  Wrapped DataFrame to write.
680 
681  """
682  additionalData = butlerLocation.getAdditionalData()
683  locations = butlerLocation.getLocations()
684  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
685  logLoc = LogicalLocation(locationString, additionalData)
686  filename = logLoc.locString()
687  obj.write(filename)
688 
689 
690 def writeYamlStorage(butlerLocation, obj):
691  """Writes an object to a YAML file specified by ButlerLocation.
692 
693  Parameters
694  ----------
695  butlerLocation : ButlerLocation
696  The location for the object to be written.
697  obj : object instance
698  The object to be written.
699  """
700  additionalData = butlerLocation.getAdditionalData()
701  locations = butlerLocation.getLocations()
702  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
703  logLoc = LogicalLocation(locationString, additionalData)
704  with open(logLoc.locString(), "w") as outfile:
705  yaml.dump(obj, outfile)
706 
707 
708 def readPickleStorage(butlerLocation):
709  """Read an object from a pickle file specified by ButlerLocation.
710 
711  Parameters
712  ----------
713  butlerLocation : ButlerLocation
714  The location for the object(s) to be read.
715 
716  Returns
717  -------
718  A list of objects as described by the butler location. One item for
719  each location in butlerLocation.getLocations()
720  """
721  # Create a list of Storages for the item.
722  results = []
723  additionalData = butlerLocation.getAdditionalData()
724  for locationString in butlerLocation.getLocations():
725  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
726  logLoc = LogicalLocation(locStringWithRoot, additionalData)
727  if not os.path.exists(logLoc.locString()):
728  raise RuntimeError("No such pickle file: " + logLoc.locString())
729  with open(logLoc.locString(), "rb") as infile:
730  # py3: We have to specify encoding since some files were written
731  # by python2, and 'latin1' manages that conversion safely. See:
732  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
733  if sys.version_info.major >= 3:
734  finalItem = pickle.load(infile, encoding="latin1")
735  else:
736  finalItem = pickle.load(infile)
737  results.append(finalItem)
738  return results
739 
740 
741 def writePickleStorage(butlerLocation, obj):
742  """Writes an object to a pickle file specified by ButlerLocation.
743 
744  Parameters
745  ----------
746  butlerLocation : ButlerLocation
747  The location for the object to be written.
748  obj : object instance
749  The object to be written.
750  """
751  additionalData = butlerLocation.getAdditionalData()
752  locations = butlerLocation.getLocations()
753  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
754  logLoc = LogicalLocation(locationString, additionalData)
755  with open(logLoc.locString(), "wb") as outfile:
756  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
757 
758 
759 def readFitsCatalogStorage(butlerLocation):
760  """Read a catalog from a FITS table specified by ButlerLocation.
761 
762  Parameters
763  ----------
764  butlerLocation : ButlerLocation
765  The location for the object(s) to be read.
766 
767  Returns
768  -------
769  A list of objects as described by the butler location. One item for
770  each location in butlerLocation.getLocations()
771  """
772  pythonType = butlerLocation.getPythonType()
773  if pythonType is not None:
774  if isinstance(pythonType, basestring):
775  pythonType = doImport(pythonType)
776  results = []
777  additionalData = butlerLocation.getAdditionalData()
778  for locationString in butlerLocation.getLocations():
779  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
780  logLoc = LogicalLocation(locStringWithRoot, additionalData)
781  if not os.path.exists(logLoc.locString()):
782  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
783  kwds = {}
784  if additionalData.exists("hdu"):
785  kwds["hdu"] = additionalData.getInt("hdu")
786  if additionalData.exists("flags"):
787  kwds["flags"] = additionalData.getInt("flags")
788  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
789  results.append(finalItem)
790  return results
791 
792 
793 def writeFitsCatalogStorage(butlerLocation, obj):
794  """Writes a catalog to a FITS table specified by ButlerLocation.
795 
796  Parameters
797  ----------
798  butlerLocation : ButlerLocation
799  The location for the object to be written.
800  obj : object instance
801  The object to be written.
802  """
803  additionalData = butlerLocation.getAdditionalData()
804  locations = butlerLocation.getLocations()
805  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
806  logLoc = LogicalLocation(locationString, additionalData)
807  if additionalData.exists("flags"):
808  kwds = dict(flags=additionalData.getInt("flags"))
809  else:
810  kwds = {}
811  obj.writeFits(logLoc.locString(), **kwds)
812 
813 
814 def readMatplotlibStorage(butlerLocation):
815  """Read from a butlerLocation (always fails for this storage type).
816 
817  Parameters
818  ----------
819  butlerLocation : ButlerLocation
820  The location for the object(s) to be read.
821 
822  Returns
823  -------
824  A list of objects as described by the butler location. One item for
825  each location in butlerLocation.getLocations()
826  """
827  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
828 
829 
830 def writeMatplotlibStorage(butlerLocation, obj):
831  """Writes a matplotlib.figure.Figure to a location, using the template's
832  filename suffix to infer the file format.
833 
834  Parameters
835  ----------
836  butlerLocation : ButlerLocation
837  The location for the object to be written.
838  obj : matplotlib.figure.Figure
839  The object to be written.
840  """
841  additionalData = butlerLocation.getAdditionalData()
842  locations = butlerLocation.getLocations()
843  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
844  logLoc = LogicalLocation(locationString, additionalData)
845  # SafeFilename appends a random suffix, which corrupts the extension
846  # matplotlib uses to guess the file format.
847  # Instead, we extract the extension from the original location
848  # and pass that as the format directly.
849  _, ext = os.path.splitext(locations[0])
850  if ext:
851  ext = ext[1:] # strip off leading '.'
852  else:
853  # If there is no extension, we let matplotlib fall back to its
854  # default.
855  ext = None
856  obj.savefig(logLoc.locString(), format=ext)
857 
858 
859 def readPafStorage(butlerLocation):
860  """Read a policy from a PAF file specified by a ButlerLocation.
861 
862  Parameters
863  ----------
864  butlerLocation : ButlerLocation
865  The location for the object(s) to be read.
866 
867  Returns
868  -------
869  A list of objects as described by the butler location. One item for
870  each location in butlerLocation.getLocations()
871  """
872  results = []
873  for locationString in butlerLocation.getLocations():
874  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
875  butlerLocation.getAdditionalData())
876  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
877  results.append(finalItem)
878  return results
879 
880 
881 def readYamlStorage(butlerLocation):
882  """Read an object from a YAML file specified by a butlerLocation.
883 
884  Parameters
885  ----------
886  butlerLocation : ButlerLocation
887  The location for the object(s) to be read.
888 
889  Returns
890  -------
891  A list of objects as described by the butler location. One item for
892  each location in butlerLocation.getLocations()
893  """
894  results = []
895  for locationString in butlerLocation.getLocations():
896  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
897  butlerLocation.getAdditionalData())
898  if not os.path.exists(logLoc.locString()):
899  raise RuntimeError("No such YAML file: " + logLoc.locString())
900  # Butler Gen2 repository configurations are handled specially
901  if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
902  finalItem = Policy(filePath=logLoc.locString())
903  else:
904  with open(logLoc.locString(), "rb") as infile:
905  finalItem = yaml.load(infile)
906  results.append(finalItem)
907  return results
908 
909 
910 def readBoostStorage(butlerLocation):
911  """Read an object from a boost::serialization file.
912 
913  Parameters
914  ----------
915  butlerLocation : ButlerLocation
916  The location for the object(s) to be read.
917 
918  Returns
919  -------
920  A list of objects as described by the butler location. One item for
921  each location in butlerLocation.getLocations()
922  """
923  results = []
924  additionalData = butlerLocation.getAdditionalData()
925  for locationString in butlerLocation.getLocations():
926  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
927  butlerLocation.getAdditionalData())
928  storageList = StorageList()
929  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
930  storageList.append(storage)
931  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
932  additionalData)
933  results.append(finalItem)
934  return results
935 
936 
937 def writeBoostStorage(butlerLocation, obj):
938  """Writes an object via boost::serialization.
939 
940  Parameters
941  ----------
942  butlerLocation : ButlerLocation
943  The location for the object to be written.
944  obj : object instance
945  The object to be written.
946  """
947  additionalData = butlerLocation.getAdditionalData()
948  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
949  with SafeFilename(location) as locationString:
950  logLoc = LogicalLocation(locationString, additionalData)
951  # Create a list of Storages for the item.
952  storageList = StorageList()
953  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
954  storageList.append(storage)
955  # Persist the item.
956  if hasattr(obj, '__deref__'):
957  # We have a smart pointer, so dereference it.
958  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
959  else:
960  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
961 
962 
963 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
964 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
965 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
966 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
967 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
968 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
969 PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
970 PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
971 
972 Storage.registerStorageClass(scheme='', cls=PosixStorage)
973 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeYamlStorage(butlerLocation, obj)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)