lsst.daf.persistence  16.0-3-g3806c63
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import urllib.parse
30 import glob
31 import shutil
32 import yaml
33 
34 from . import (LogicalLocation, Persistence, Policy, StorageList,
35  StorageInterface, Storage, ButlerLocation,
36  NoRepositroyAtRoot, RepositoryCfg, doImport)
37 from lsst.log import Log
38 import lsst.pex.policy as pexPolicy
39 from .safeFileIo import SafeFilename, safeMakeDir
40 # Make YAML representers for daf_base classes available
41 from . import baseYaml # noqa F401
42 
43 
44 __all__ = ["PosixStorage"]
45 
46 
48  """Defines the interface for a storage location on the local filesystem.
49 
50  Parameters
51  ----------
52  uri : string
53  URI or path that is used as the storage location.
54  create : bool
55  If True a new repository will be created at the root location if it
56  does not exist. If False then a new repository will not be created.
57 
58  Raises
59  ------
60  NoRepositroyAtRoot
61  If create is False and a repository does not exist at the root
62  specified by uri then NoRepositroyAtRoot is raised.
63  """
64 
65  def __init__(self, uri, create):
66  self.log = Log.getLogger("daf.persistence.butler")
67  self.root = self._pathFromURI(uri)
68  if self.root and not os.path.exists(self.root):
69  if not create:
70  raise NoRepositroyAtRoot("No repository at {}".format(uri))
71  safeMakeDir(self.root)
72 
74 
75  @staticmethod
77  # Always use an empty Persistence policy until we can get rid of it
78  persistencePolicy = pexPolicy.Policy()
79  return Persistence.getPersistence(persistencePolicy)
80 
81  def __repr__(self):
82  return 'PosixStorage(root=%s)' % self.root
83 
84  @staticmethod
85  def _pathFromURI(uri):
86  """Get the path part of the URI"""
87  return urllib.parse.urlparse(uri).path
88 
89  @staticmethod
90  def relativePath(fromPath, toPath):
91  """Get a relative path from a location to a location.
92 
93  Parameters
94  ----------
95  fromPath : string
96  A path at which to start. It can be a relative path or an
97  absolute path.
98  toPath : string
99  A target location. It can be a relative path or an absolute path.
100 
101  Returns
102  -------
103  string
104  A relative path that describes the path from fromPath to toPath.
105  """
106  fromPath = os.path.realpath(fromPath)
107  return os.path.relpath(toPath, fromPath)
108 
109  @staticmethod
110  def absolutePath(fromPath, relativePath):
111  """Get an absolute path for the path from fromUri to toUri
112 
113  Parameters
114  ----------
115  fromPath : the starting location
116  A location at which to start. It can be a relative path or an
117  absolute path.
118  relativePath : the location relative to fromPath
119  A relative path.
120 
121  Returns
122  -------
123  string
124  Path that is an absolute path representation of fromPath +
125  relativePath, if one exists. If relativePath is absolute or if
126  fromPath is not related to relativePath then relativePath will be
127  returned.
128  """
129  if os.path.isabs(relativePath):
130  return relativePath
131  fromPath = os.path.realpath(fromPath)
132  return os.path.normpath(os.path.join(fromPath, relativePath))
133 
134  @staticmethod
136  """Get a persisted RepositoryCfg
137 
138  Parameters
139  ----------
140  uri : URI or path to a RepositoryCfg
141  Description
142 
143  Returns
144  -------
145  A RepositoryCfg instance or None
146  """
147  storage = Storage.makeFromURI(uri)
148  location = ButlerLocation(pythonType=RepositoryCfg,
149  cppType=None,
150  storageName=None,
151  locationList='repositoryCfg.yaml',
152  dataId={},
153  mapper=None,
154  storage=storage,
155  usedDataId=None,
156  datasetType=None)
157  return storage.read(location)
158 
159  @staticmethod
160  def putRepositoryCfg(cfg, loc=None):
161  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
162  location = ButlerLocation(pythonType=RepositoryCfg,
163  cppType=None,
164  storageName=None,
165  locationList='repositoryCfg.yaml',
166  dataId={},
167  mapper=None,
168  storage=storage,
169  usedDataId=None,
170  datasetType=None)
171  storage.write(location, cfg)
172 
173  @staticmethod
174  def getMapperClass(root):
175  """Get the mapper class associated with a repository root.
176 
177  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
178  new code and repositories; they should use the Repository parentCfg mechanism.
179 
180  Parameters
181  ----------
182  root : string
183  The location of a persisted ReositoryCfg is (new style repos), or
184  the location where a _mapper file is (old style repos).
185 
186  Returns
187  -------
188  A class object or a class instance, depending on the state of the
189  mapper when the repository was created.
190  """
191  if not (root):
192  return None
193 
194  cfg = PosixStorage.getRepositoryCfg(root)
195  if cfg is not None:
196  return cfg.mapper
197 
198  # Find a "_mapper" file containing the mapper class name
199  basePath = root
200  mapperFile = "_mapper"
201  while not os.path.exists(os.path.join(basePath, mapperFile)):
202  # Break abstraction by following _parent links from CameraMapper
203  if os.path.exists(os.path.join(basePath, "_parent")):
204  basePath = os.path.join(basePath, "_parent")
205  else:
206  mapperFile = None
207  break
208 
209  if mapperFile is not None:
210  mapperFile = os.path.join(basePath, mapperFile)
211 
212  # Read the name of the mapper class and instantiate it
213  with open(mapperFile, "r") as f:
214  mapperName = f.readline().strip()
215  components = mapperName.split(".")
216  if len(components) <= 1:
217  raise RuntimeError("Unqualified mapper name %s in %s" %
218  (mapperName, mapperFile))
219  pkg = importlib.import_module(".".join(components[:-1]))
220  return getattr(pkg, components[-1])
221 
222  return None
223 
224  @staticmethod
226  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
227  symlink.
228 
229  Parameters
230  ----------
231  root : string
232  A path to the folder on the local filesystem.
233 
234  Returns
235  -------
236  string or None
237  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
238  symlink at root.
239  """
240  linkpath = os.path.join(root, '_parent')
241  if os.path.exists(linkpath):
242  try:
243  return os.readlink(os.path.join(root, '_parent'))
244  except OSError:
245  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
246  # location. Allow that; return the path of that folder.
247  return os.path.join(root, '_parent')
248  return None
249 
250  def write(self, butlerLocation, obj):
251  """Writes an object to a location and persistence format specified by
252  ButlerLocation
253 
254  Parameters
255  ----------
256  butlerLocation : ButlerLocation
257  The location & formatting for the object to be written.
258  obj : object instance
259  The object to be written.
260  """
261  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
262 
263  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
264  if not writeFormatter:
265  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
266  if writeFormatter:
267  writeFormatter(butlerLocation, obj)
268  return
269 
270  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
271 
272  def read(self, butlerLocation):
273  """Read from a butlerLocation.
274 
275  Parameters
276  ----------
277  butlerLocation : ButlerLocation
278  The location & formatting for the object(s) to be read.
279 
280  Returns
281  -------
282  A list of objects as described by the butler location. One item for
283  each location in butlerLocation.getLocations()
284  """
285  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
286  if not readFormatter:
287  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
288  if readFormatter:
289  return readFormatter(butlerLocation)
290 
291  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
292 
293  def butlerLocationExists(self, location):
294  """Implementation of PosixStorage.exists for ButlerLocation objects.
295  """
296  storageName = location.getStorageName()
297  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
298  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
299  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
300  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
301  return False
302  for locationString in location.getLocations():
303  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
304  obj = self.instanceSearch(path=logLoc)
305  if obj:
306  return True
307  return False
308 
309  def exists(self, location):
310  """Check if location exists.
311 
312  Parameters
313  ----------
314  location : ButlerLocation or string
315  A a string or a ButlerLocation that describes the location of an
316  object in this storage.
317 
318  Returns
319  -------
320  bool
321  True if exists, else False.
322  """
323  if isinstance(location, ButlerLocation):
324  return self.butlerLocationExists(location)
325 
326  obj = self.instanceSearch(path=location)
327  return bool(obj)
328 
329  def locationWithRoot(self, location):
330  """Get the full path to the location.
331 
332  :param location:
333  :return:
334  """
335  return os.path.join(self.root, location)
336 
337  @staticmethod
338  def v1RepoExists(root):
339  """Test if a Version 1 Repository exists.
340 
341  Version 1 Repositories only exist in posix storages, do not have a
342  RepositoryCfg file, and contain either a registry.sqlite3 file, a
343  _mapper file, or a _parent link.
344 
345  Parameters
346  ----------
347  root : string
348  A path to a folder on the local filesystem.
349 
350  Returns
351  -------
352  bool
353  True if the repository at root exists, else False.
354  """
355  return os.path.exists(root) and (
356  os.path.exists(os.path.join(root, "registry.sqlite3")) or
357  os.path.exists(os.path.join(root, "_mapper")) or
358  os.path.exists(os.path.join(root, "_parent"))
359  )
360 
361  def copyFile(self, fromLocation, toLocation):
362  """Copy a file from one location to another on the local filesystem.
363 
364  Parameters
365  ----------
366  fromLocation : path
367  Path and name of existing file.
368  toLocation : path
369  Path and name of new file.
370 
371  Returns
372  -------
373  None
374  """
375  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
376 
377  def getLocalFile(self, path):
378  """Get a handle to a local copy of the file, downloading it to a
379  temporary if needed.
380 
381  Parameters
382  ----------
383  A path the the file in storage, relative to root.
384 
385  Returns
386  -------
387  A handle to a local copy of the file. If storage is remote it will be
388  a temporary file. If storage is local it may be the original file or
389  a temporary file. The file name can be gotten via the 'name' property
390  of the returned object.
391  """
392  p = os.path.join(self.root, path)
393  try:
394  return open(p)
395  except IOError as e:
396  if e.errno == 2: # 'No such file or directory'
397  return None
398  else:
399  raise e
400 
401  def instanceSearch(self, path):
402  """Search for the given path in this storage instance.
403 
404  If the path contains an HDU indicator (a number in brackets before the
405  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
406  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
407  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
408 
409  Parameters
410  ----------
411  path : string
412  A filename (and optionally prefix path) to search for within root.
413 
414  Returns
415  -------
416  string or None
417  The location that was found, or None if no location was found.
418  """
419  return self.search(self.root, path)
420 
421  @staticmethod
422  def search(root, path, searchParents=False):
423  """Look for the given path in the current root.
424 
425  Also supports searching for the path in Butler v1 repositories by
426  following the Butler v1 _parent symlink
427 
428  If the path contains an HDU indicator (a number in brackets, e.g.
429  'foo.fits[1]', this will be stripped when searching and so
430  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
431  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
432 
433  Parameters
434  ----------
435  root : string
436  The path to the root directory.
437  path : string
438  The path to the file within the root directory.
439  searchParents : bool, optional
440  For Butler v1 repositories only, if true and a _parent symlink
441  exists, then the directory at _parent will be searched if the file
442  is not found in the root repository. Will continue searching the
443  parent of the parent until the file is found or no additional
444  parent exists.
445 
446  Returns
447  -------
448  string or None
449  The location that was found, or None if no location was found.
450  """
451  # Separate path into a root-equivalent prefix (in dir) and the rest
452  # (left in path)
453  rootDir = root
454  # First remove trailing slashes (#2527)
455  while len(rootDir) > 1 and rootDir[-1] == '/':
456  rootDir = rootDir[:-1]
457 
458  if path.startswith(rootDir + "/"):
459  # Common case; we have the same root prefix string
460  path = path[len(rootDir + '/'):]
461  pathPrefix = rootDir
462  elif rootDir == "/" and path.startswith("/"):
463  path = path[1:]
464  pathPrefix = None
465  else:
466  # Search for prefix that is the same as root
467  pathPrefix = os.path.dirname(path)
468  while pathPrefix != "" and pathPrefix != "/":
469  if os.path.realpath(pathPrefix) == os.path.realpath(root):
470  break
471  pathPrefix = os.path.dirname(pathPrefix)
472  if pathPrefix == "/":
473  path = path[1:]
474  elif pathPrefix != "":
475  path = path[len(pathPrefix)+1:]
476 
477  # Now search for the path in the root or its parents
478  # Strip off any cfitsio bracketed extension if present
479  strippedPath = path
480  pathStripped = None
481  firstBracket = path.find("[")
482  if firstBracket != -1:
483  strippedPath = path[:firstBracket]
484  pathStripped = path[firstBracket:]
485 
486  dir = rootDir
487  while True:
488  paths = glob.glob(os.path.join(dir, strippedPath))
489  if len(paths) > 0:
490  if pathPrefix != rootDir:
491  paths = [p[len(rootDir+'/'):] for p in paths]
492  if pathStripped is not None:
493  paths = [p + pathStripped for p in paths]
494  return paths
495  if searchParents:
496  dir = os.path.join(dir, "_parent")
497  if not os.path.exists(dir):
498  return None
499  else:
500  return None
501 
502  @staticmethod
503  def storageExists(uri):
504  """Ask if a storage at the location described by uri exists
505 
506  Parameters
507  ----------
508  root : string
509  URI to the the root location of the storage
510 
511  Returns
512  -------
513  bool
514  True if the storage exists, false if not
515  """
516  return os.path.exists(PosixStorage._pathFromURI(uri))
517 
518 
519 def readConfigStorage(butlerLocation):
520  """Read an lsst.pex.config.Config from a butlerLocation.
521 
522  Parameters
523  ----------
524  butlerLocation : ButlerLocation
525  The location for the object(s) to be read.
526 
527  Returns
528  -------
529  A list of objects as described by the butler location. One item for
530  each location in butlerLocation.getLocations()
531  """
532  results = []
533  for locationString in butlerLocation.getLocations():
534  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
535  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
536  if not os.path.exists(logLoc.locString()):
537  raise RuntimeError("No such config file: " + logLoc.locString())
538  pythonType = butlerLocation.getPythonType()
539  if pythonType is not None:
540  if isinstance(pythonType, basestring):
541  pythonType = doImport(pythonType)
542  finalItem = pythonType()
543  finalItem.load(logLoc.locString())
544  results.append(finalItem)
545  return results
546 
547 
548 def writeConfigStorage(butlerLocation, obj):
549  """Writes an lsst.pex.config.Config object to a location specified by
550  ButlerLocation.
551 
552  Parameters
553  ----------
554  butlerLocation : ButlerLocation
555  The location for the object to be written.
556  obj : object instance
557  The object to be written.
558  """
559  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
560  with SafeFilename(filename) as locationString:
561  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
562  obj.save(logLoc.locString())
563 
564 
565 def readFitsStorage(butlerLocation):
566  """Read a FITS image from a butlerLocation.
567 
568  Parameters
569  ----------
570  butlerLocation : ButlerLocation
571  The location for the object(s) to be read.
572 
573  Returns
574  -------
575  A list of objects as described by the butler location. One item for
576  each location in butlerLocation.getLocations()
577  """
578  results = []
579  for locationString in butlerLocation.getLocations():
580  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
581  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
582  storageList = StorageList()
583  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
584  logLoc)
585  storageList.append(storage)
586  finalItem = PosixStorage.getPersistence().unsafeRetrieve(
587  butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
588  results.append(finalItem)
589  return results
590 
591 
592 def writeFitsStorage(butlerLocation, obj):
593  """Writes an object to a FITS file specified by ButlerLocation.
594 
595  Parameters
596  ----------
597  butlerLocation : ButlerLocation
598  The location for the object to be written.
599  obj : object instance
600  The object to be written.
601  """
602  location = butlerLocation.getLocations()[0]
603  with SafeFilename(os.path.join(butlerLocation.getStorage().root, location)) as locationString:
604  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
605  # Create a list of Storages for the item.
606  storageList = StorageList()
607  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
608  storageList.append(storage)
609  persistence = PosixStorage.getPersistence()
610  if hasattr(obj, '__deref__'):
611  # We have a smart pointer, so dereference it.
612  persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
613  else:
614  persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
615 
616 
617 def readParquetStorage(butlerLocation):
618  """Read a catalog from a Parquet file specified by ButlerLocation.
619 
620  The object returned by this is expected to be a subtype
621  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
622  that allows for lazy loading of the data.
623 
624  Parameters
625  ----------
626  butlerLocation : ButlerLocation
627  The location for the object(s) to be read.
628 
629  Returns
630  -------
631  A list of objects as described by the butler location. One item for
632  each location in butlerLocation.getLocations()
633  """
634  results = []
635  additionalData = butlerLocation.getAdditionalData()
636 
637  for locationString in butlerLocation.getLocations():
638  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
639  logLoc = LogicalLocation(locStringWithRoot, additionalData)
640  if not os.path.exists(logLoc.locString()):
641  raise RuntimeError("No such parquet file: " + logLoc.locString())
642 
643  pythonType = butlerLocation.getPythonType()
644  if pythonType is not None:
645  if isinstance(pythonType, basestring):
646  pythonType = doImport(pythonType)
647 
648  filename = logLoc.locString()
649 
650  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
651  # filename should be the first kwarg, but being explicit here.
652  results.append(pythonType(filename=filename))
653 
654  return results
655 
656 
657 def writeParquetStorage(butlerLocation, obj):
658  """Writes pandas dataframe to parquet file.
659 
660  Parameters
661  ----------
662  butlerLocation : ButlerLocation
663  The location for the object(s) to be read.
664  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
665  Wrapped DataFrame to write.
666 
667  """
668  additionalData = butlerLocation.getAdditionalData()
669  locations = butlerLocation.getLocations()
670  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
671  logLoc = LogicalLocation(locationString, additionalData)
672  filename = logLoc.locString()
673  obj.write(filename)
674 
675 
676 def writeYamlStorage(butlerLocation, obj):
677  """Writes an object to a YAML file specified by ButlerLocation.
678 
679  Parameters
680  ----------
681  butlerLocation : ButlerLocation
682  The location for the object to be written.
683  obj : object instance
684  The object to be written.
685  """
686  additionalData = butlerLocation.getAdditionalData()
687  locations = butlerLocation.getLocations()
688  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
689  logLoc = LogicalLocation(locationString, additionalData)
690  with open(logLoc.locString(), "w") as outfile:
691  yaml.dump(obj, outfile)
692 
693 
694 def readPickleStorage(butlerLocation):
695  """Read an object from a pickle file specified by ButlerLocation.
696 
697  Parameters
698  ----------
699  butlerLocation : ButlerLocation
700  The location for the object(s) to be read.
701 
702  Returns
703  -------
704  A list of objects as described by the butler location. One item for
705  each location in butlerLocation.getLocations()
706  """
707  # Create a list of Storages for the item.
708  results = []
709  additionalData = butlerLocation.getAdditionalData()
710  for locationString in butlerLocation.getLocations():
711  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
712  logLoc = LogicalLocation(locStringWithRoot, additionalData)
713  if not os.path.exists(logLoc.locString()):
714  raise RuntimeError("No such pickle file: " + logLoc.locString())
715  with open(logLoc.locString(), "rb") as infile:
716  # py3: We have to specify encoding since some files were written
717  # by python2, and 'latin1' manages that conversion safely. See:
718  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
719  if sys.version_info.major >= 3:
720  finalItem = pickle.load(infile, encoding="latin1")
721  else:
722  finalItem = pickle.load(infile)
723  results.append(finalItem)
724  return results
725 
726 
727 def writePickleStorage(butlerLocation, obj):
728  """Writes an object to a pickle file specified by ButlerLocation.
729 
730  Parameters
731  ----------
732  butlerLocation : ButlerLocation
733  The location for the object to be written.
734  obj : object instance
735  The object to be written.
736  """
737  additionalData = butlerLocation.getAdditionalData()
738  locations = butlerLocation.getLocations()
739  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
740  logLoc = LogicalLocation(locationString, additionalData)
741  with open(logLoc.locString(), "wb") as outfile:
742  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
743 
744 
745 def readFitsCatalogStorage(butlerLocation):
746  """Read a catalog from a FITS table specified by ButlerLocation.
747 
748  Parameters
749  ----------
750  butlerLocation : ButlerLocation
751  The location for the object(s) to be read.
752 
753  Returns
754  -------
755  A list of objects as described by the butler location. One item for
756  each location in butlerLocation.getLocations()
757  """
758  pythonType = butlerLocation.getPythonType()
759  if pythonType is not None:
760  if isinstance(pythonType, basestring):
761  pythonType = doImport(pythonType)
762  results = []
763  additionalData = butlerLocation.getAdditionalData()
764  for locationString in butlerLocation.getLocations():
765  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
766  logLoc = LogicalLocation(locStringWithRoot, additionalData)
767  if not os.path.exists(logLoc.locString()):
768  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
769  kwds = {}
770  if additionalData.exists("hdu"):
771  kwds["hdu"] = additionalData.getInt("hdu")
772  if additionalData.exists("flags"):
773  kwds["flags"] = additionalData.getInt("flags")
774  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
775  results.append(finalItem)
776  return results
777 
778 
779 def writeFitsCatalogStorage(butlerLocation, obj):
780  """Writes a catalog to a FITS table specified by ButlerLocation.
781 
782  Parameters
783  ----------
784  butlerLocation : ButlerLocation
785  The location for the object to be written.
786  obj : object instance
787  The object to be written.
788  """
789  additionalData = butlerLocation.getAdditionalData()
790  locations = butlerLocation.getLocations()
791  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
792  logLoc = LogicalLocation(locationString, additionalData)
793  if additionalData.exists("flags"):
794  kwds = dict(flags=additionalData.getInt("flags"))
795  else:
796  kwds = {}
797  obj.writeFits(logLoc.locString(), **kwds)
798 
799 
800 def readMatplotlibStorage(butlerLocation):
801  """Read from a butlerLocation (always fails for this storage type).
802 
803  Parameters
804  ----------
805  butlerLocation : ButlerLocation
806  The location for the object(s) to be read.
807 
808  Returns
809  -------
810  A list of objects as described by the butler location. One item for
811  each location in butlerLocation.getLocations()
812  """
813  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
814 
815 
816 def writeMatplotlibStorage(butlerLocation, obj):
817  """Writes a matplotlib.figure.Figure to a location, using the template's
818  filename suffix to infer the file format.
819 
820  Parameters
821  ----------
822  butlerLocation : ButlerLocation
823  The location for the object to be written.
824  obj : matplotlib.figure.Figure
825  The object to be written.
826  """
827  additionalData = butlerLocation.getAdditionalData()
828  locations = butlerLocation.getLocations()
829  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
830  logLoc = LogicalLocation(locationString, additionalData)
831  # SafeFilename appends a random suffix, which corrupts the extension
832  # matplotlib uses to guess the file format.
833  # Instead, we extract the extension from the original location
834  # and pass that as the format directly.
835  _, ext = os.path.splitext(locations[0])
836  if ext:
837  ext = ext[1:] # strip off leading '.'
838  else:
839  # If there is no extension, we let matplotlib fall back to its
840  # default.
841  ext = None
842  obj.savefig(logLoc.locString(), format=ext)
843 
844 
845 def readPafStorage(butlerLocation):
846  """Read a policy from a PAF file specified by a ButlerLocation.
847 
848  Parameters
849  ----------
850  butlerLocation : ButlerLocation
851  The location for the object(s) to be read.
852 
853  Returns
854  -------
855  A list of objects as described by the butler location. One item for
856  each location in butlerLocation.getLocations()
857  """
858  results = []
859  for locationString in butlerLocation.getLocations():
860  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
861  butlerLocation.getAdditionalData())
862  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
863  results.append(finalItem)
864  return results
865 
866 
867 def readYamlStorage(butlerLocation):
868  """Read an object from a YAML file specified by a butlerLocation.
869 
870  Parameters
871  ----------
872  butlerLocation : ButlerLocation
873  The location for the object(s) to be read.
874 
875  Returns
876  -------
877  A list of objects as described by the butler location. One item for
878  each location in butlerLocation.getLocations()
879  """
880  results = []
881  for locationString in butlerLocation.getLocations():
882  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
883  butlerLocation.getAdditionalData())
884  if not os.path.exists(logLoc.locString()):
885  raise RuntimeError("No such YAML file: " + logLoc.locString())
886  # Butler Gen2 repository configurations are handled specially
887  if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
888  finalItem = Policy(filePath=logLoc.locString())
889  else:
890  with open(logLoc.locString(), "rb") as infile:
891  finalItem = yaml.load(infile)
892  results.append(finalItem)
893  return results
894 
895 
896 def readBoostStorage(butlerLocation):
897  """Read an object from a boost::serialization file.
898 
899  Parameters
900  ----------
901  butlerLocation : ButlerLocation
902  The location for the object(s) to be read.
903 
904  Returns
905  -------
906  A list of objects as described by the butler location. One item for
907  each location in butlerLocation.getLocations()
908  """
909  results = []
910  additionalData = butlerLocation.getAdditionalData()
911  for locationString in butlerLocation.getLocations():
912  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
913  butlerLocation.getAdditionalData())
914  storageList = StorageList()
915  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
916  storageList.append(storage)
917  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
918  additionalData)
919  results.append(finalItem)
920  return results
921 
922 
923 def writeBoostStorage(butlerLocation, obj):
924  """Writes an object via boost::serialization.
925 
926  Parameters
927  ----------
928  butlerLocation : ButlerLocation
929  The location for the object to be written.
930  obj : object instance
931  The object to be written.
932  """
933  additionalData = butlerLocation.getAdditionalData()
934  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
935  with SafeFilename(location) as locationString:
936  logLoc = LogicalLocation(locationString, additionalData)
937  # Create a list of Storages for the item.
938  storageList = StorageList()
939  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
940  storageList.append(storage)
941  # Persist the item.
942  if hasattr(obj, '__deref__'):
943  # We have a smart pointer, so dereference it.
944  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
945  else:
946  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
947 
948 
949 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
950 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
951 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
952 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
953 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
954 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
955 PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
956 PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
957 PosixStorage.registerFormatters("BoostStorage", readFitsStorage, writeFitsStorage)
958 
959 Storage.registerStorageClass(scheme='', cls=PosixStorage)
960 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeYamlStorage(butlerLocation, obj)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)