lsst.daf.persistence  15.0-4-g45f767a
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import urllib.parse
30 import glob
31 import shutil
32 
33 from . import (LogicalLocation, Persistence, Policy, StorageList,
34  StorageInterface, Storage, ButlerLocation,
35  NoRepositroyAtRoot, RepositoryCfg, doImport)
36 from lsst.log import Log
37 import lsst.pex.policy as pexPolicy
38 from .safeFileIo import SafeFilename, safeMakeDir
39 
40 
41 __all__ = ["PosixStorage"]
42 
43 
45  """Defines the interface for a storage location on the local filesystem.
46 
47  Parameters
48  ----------
49  uri : string
50  URI or path that is used as the storage location.
51  create : bool
52  If True a new repository will be created at the root location if it
53  does not exist. If False then a new repository will not be created.
54 
55  Raises
56  ------
57  NoRepositroyAtRoot
58  If create is False and a repository does not exist at the root
59  specified by uri then NoRepositroyAtRoot is raised.
60  """
61 
62  def __init__(self, uri, create):
63  self.log = Log.getLogger("daf.persistence.butler")
64  self.root = self._pathFromURI(uri)
65  if self.root and not os.path.exists(self.root):
66  if not create:
67  raise NoRepositroyAtRoot("No repository at {}".format(uri))
68  safeMakeDir(self.root)
69 
71 
72  @staticmethod
74  # Always use an empty Persistence policy until we can get rid of it
75  persistencePolicy = pexPolicy.Policy()
76  return Persistence.getPersistence(persistencePolicy)
77 
78  def __repr__(self):
79  return 'PosixStorage(root=%s)' % self.root
80 
81  @staticmethod
82  def _pathFromURI(uri):
83  """Get the path part of the URI"""
84  return urllib.parse.urlparse(uri).path
85 
86  @staticmethod
87  def relativePath(fromPath, toPath):
88  """Get a relative path from a location to a location.
89 
90  Parameters
91  ----------
92  fromPath : string
93  A path at which to start. It can be a relative path or an
94  absolute path.
95  toPath : string
96  A target location. It can be a relative path or an absolute path.
97 
98  Returns
99  -------
100  string
101  A relative path that describes the path from fromPath to toPath.
102  """
103  fromPath = os.path.realpath(fromPath)
104  return os.path.relpath(toPath, fromPath)
105 
106  @staticmethod
107  def absolutePath(fromPath, relativePath):
108  """Get an absolute path for the path from fromUri to toUri
109 
110  Parameters
111  ----------
112  fromPath : the starting location
113  A location at which to start. It can be a relative path or an
114  absolute path.
115  relativePath : the location relative to fromPath
116  A relative path.
117 
118  Returns
119  -------
120  string
121  Path that is an absolute path representation of fromPath +
122  relativePath, if one exists. If relativePath is absolute or if
123  fromPath is not related to relativePath then relativePath will be
124  returned.
125  """
126  if os.path.isabs(relativePath):
127  return relativePath
128  fromPath = os.path.realpath(fromPath)
129  return os.path.normpath(os.path.join(fromPath, relativePath))
130 
131  @staticmethod
133  """Get a persisted RepositoryCfg
134 
135  Parameters
136  ----------
137  uri : URI or path to a RepositoryCfg
138  Description
139 
140  Returns
141  -------
142  A RepositoryCfg instance or None
143  """
144  storage = Storage.makeFromURI(uri)
145  location = ButlerLocation(pythonType=RepositoryCfg,
146  cppType=None,
147  storageName=None,
148  locationList='repositoryCfg.yaml',
149  dataId={},
150  mapper=None,
151  storage=storage,
152  usedDataId=None,
153  datasetType=None)
154  return storage.read(location)
155 
156  @staticmethod
157  def putRepositoryCfg(cfg, loc=None):
158  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
159  location = ButlerLocation(pythonType=RepositoryCfg,
160  cppType=None,
161  storageName=None,
162  locationList='repositoryCfg.yaml',
163  dataId={},
164  mapper=None,
165  storage=storage,
166  usedDataId=None,
167  datasetType=None)
168  storage.write(location, cfg)
169 
170  @staticmethod
171  def getMapperClass(root):
172  """Get the mapper class associated with a repository root.
173 
174  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
175  new code and repositories; they should use the Repository parentCfg mechanism.
176 
177  Parameters
178  ----------
179  root : string
180  The location of a persisted ReositoryCfg is (new style repos), or
181  the location where a _mapper file is (old style repos).
182 
183  Returns
184  -------
185  A class object or a class instance, depending on the state of the
186  mapper when the repository was created.
187  """
188  if not (root):
189  return None
190 
191  cfg = PosixStorage.getRepositoryCfg(root)
192  if cfg is not None:
193  return cfg.mapper
194 
195  # Find a "_mapper" file containing the mapper class name
196  basePath = root
197  mapperFile = "_mapper"
198  while not os.path.exists(os.path.join(basePath, mapperFile)):
199  # Break abstraction by following _parent links from CameraMapper
200  if os.path.exists(os.path.join(basePath, "_parent")):
201  basePath = os.path.join(basePath, "_parent")
202  else:
203  mapperFile = None
204  break
205 
206  if mapperFile is not None:
207  mapperFile = os.path.join(basePath, mapperFile)
208 
209  # Read the name of the mapper class and instantiate it
210  with open(mapperFile, "r") as f:
211  mapperName = f.readline().strip()
212  components = mapperName.split(".")
213  if len(components) <= 1:
214  raise RuntimeError("Unqualified mapper name %s in %s" %
215  (mapperName, mapperFile))
216  pkg = importlib.import_module(".".join(components[:-1]))
217  return getattr(pkg, components[-1])
218 
219  return None
220 
221  @staticmethod
223  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
224  symlink.
225 
226  Parameters
227  ----------
228  root : string
229  A path to the folder on the local filesystem.
230 
231  Returns
232  -------
233  string or None
234  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
235  symlink at root.
236  """
237  linkpath = os.path.join(root, '_parent')
238  if os.path.exists(linkpath):
239  try:
240  return os.readlink(os.path.join(root, '_parent'))
241  except OSError:
242  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
243  # location. Allow that; return the path of that folder.
244  return os.path.join(root, '_parent')
245  return None
246 
247  def write(self, butlerLocation, obj):
248  """Writes an object to a location and persistence format specified by
249  ButlerLocation
250 
251  Parameters
252  ----------
253  butlerLocation : ButlerLocation
254  The location & formatting for the object to be written.
255  obj : object instance
256  The object to be written.
257  """
258  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
259 
260  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
261  if not writeFormatter:
262  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
263  if writeFormatter:
264  writeFormatter(butlerLocation, obj)
265  return
266 
267  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
268 
269  def read(self, butlerLocation):
270  """Read from a butlerLocation.
271 
272  Parameters
273  ----------
274  butlerLocation : ButlerLocation
275  The location & formatting for the object(s) to be read.
276 
277  Returns
278  -------
279  A list of objects as described by the butler location. One item for
280  each location in butlerLocation.getLocations()
281  """
282  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
283  if not readFormatter:
284  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
285  if readFormatter:
286  return readFormatter(butlerLocation)
287 
288  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
289 
290  def butlerLocationExists(self, location):
291  """Implementaion of PosixStorage.exists for ButlerLocation objects."""
292  storageName = location.getStorageName()
293  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
294  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage'):
295  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
296  return False
297  for locationString in location.getLocations():
298  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
299  obj = self.instanceSearch(path=logLoc)
300  if obj:
301  return True
302  return False
303 
304  def exists(self, location):
305  """Check if location exists.
306 
307  Parameters
308  ----------
309  location : ButlerLocation or string
310  A a string or a ButlerLocation that describes the location of an
311  object in this storage.
312 
313  Returns
314  -------
315  bool
316  True if exists, else False.
317  """
318  if isinstance(location, ButlerLocation):
319  return self.butlerLocationExists(location)
320 
321  obj = self.instanceSearch(path=location)
322  return bool(obj)
323 
324  def locationWithRoot(self, location):
325  """Get the full path to the location.
326 
327  :param location:
328  :return:
329  """
330  return os.path.join(self.root, location)
331 
332  @staticmethod
333  def v1RepoExists(root):
334  """Test if a Version 1 Repository exists.
335 
336  Version 1 Repositories only exist in posix storages, do not have a
337  RepositoryCfg file, and contain either a registry.sqlite3 file, a
338  _mapper file, or a _parent link.
339 
340  Parameters
341  ----------
342  root : string
343  A path to a folder on the local filesystem.
344 
345  Returns
346  -------
347  bool
348  True if the repository at root exists, else False.
349  """
350  return os.path.exists(root) and (
351  os.path.exists(os.path.join(root, "registry.sqlite3")) or
352  os.path.exists(os.path.join(root, "_mapper")) or
353  os.path.exists(os.path.join(root, "_parent"))
354  )
355 
356  def copyFile(self, fromLocation, toLocation):
357  """Copy a file from one location to another on the local filesystem.
358 
359  Parameters
360  ----------
361  fromLocation : path
362  Path and name of existing file.
363  toLocation : path
364  Path and name of new file.
365 
366  Returns
367  -------
368  None
369  """
370  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
371 
372  def getLocalFile(self, path):
373  """Get a handle to a local copy of the file, downloading it to a
374  temporary if needed.
375 
376  Parameters
377  ----------
378  A path the the file in storage, relative to root.
379 
380  Returns
381  -------
382  A handle to a local copy of the file. If storage is remote it will be
383  a temporary file. If storage is local it may be the original file or
384  a temporary file. The file name can be gotten via the 'name' property
385  of the returned object.
386  """
387  p = os.path.join(self.root, path)
388  try:
389  return open(p)
390  except IOError as e:
391  if e.errno == 2: # 'No such file or directory'
392  return None
393  else:
394  raise e
395 
396  def instanceSearch(self, path):
397  """Search for the given path in this storage instance.
398 
399  If the path contains an HDU indicator (a number in brackets before the
400  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
401  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
402  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
403 
404  Parameters
405  ----------
406  path : string
407  A filename (and optionally prefix path) to search for within root.
408 
409  Returns
410  -------
411  string or None
412  The location that was found, or None if no location was found.
413  """
414  return self.search(self.root, path)
415 
416  @staticmethod
417  def search(root, path, searchParents=False):
418  """Look for the given path in the current root.
419 
420  Also supports searching for the path in Butler v1 repositories by
421  following the Butler v1 _parent symlink
422 
423  If the path contains an HDU indicator (a number in brackets, e.g.
424  'foo.fits[1]', this will be stripped when searching and so
425  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
426  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
427 
428  Parameters
429  ----------
430  root : string
431  The path to the root directory.
432  path : string
433  The path to the file within the root directory.
434  searchParents : bool, optional
435  For Butler v1 repositories only, if true and a _parent symlink
436  exists, then the directory at _parent will be searched if the file
437  is not found in the root repository. Will continue searching the
438  parent of the parent until the file is found or no additional
439  parent exists.
440 
441  Returns
442  -------
443  string or None
444  The location that was found, or None if no location was found.
445  """
446  # Separate path into a root-equivalent prefix (in dir) and the rest
447  # (left in path)
448  rootDir = root
449  # First remove trailing slashes (#2527)
450  while len(rootDir) > 1 and rootDir[-1] == '/':
451  rootDir = rootDir[:-1]
452 
453  if path.startswith(rootDir + "/"):
454  # Common case; we have the same root prefix string
455  path = path[len(rootDir + '/'):]
456  pathPrefix = rootDir
457  elif rootDir == "/" and path.startswith("/"):
458  path = path[1:]
459  pathPrefix = None
460  else:
461  # Search for prefix that is the same as root
462  pathPrefix = os.path.dirname(path)
463  while pathPrefix != "" and pathPrefix != "/":
464  if os.path.realpath(pathPrefix) == os.path.realpath(root):
465  break
466  pathPrefix = os.path.dirname(pathPrefix)
467  if pathPrefix == "/":
468  path = path[1:]
469  elif pathPrefix != "":
470  path = path[len(pathPrefix)+1:]
471 
472  # Now search for the path in the root or its parents
473  # Strip off any cfitsio bracketed extension if present
474  strippedPath = path
475  pathStripped = None
476  firstBracket = path.find("[")
477  if firstBracket != -1:
478  strippedPath = path[:firstBracket]
479  pathStripped = path[firstBracket:]
480 
481  dir = rootDir
482  while True:
483  paths = glob.glob(os.path.join(dir, strippedPath))
484  if len(paths) > 0:
485  if pathPrefix != rootDir:
486  paths = [p[len(rootDir+'/'):] for p in paths]
487  if pathStripped is not None:
488  paths = [p + pathStripped for p in paths]
489  return paths
490  if searchParents:
491  dir = os.path.join(dir, "_parent")
492  if not os.path.exists(dir):
493  return None
494  else:
495  return None
496 
497  @staticmethod
498  def storageExists(uri):
499  """Ask if a storage at the location described by uri exists
500 
501  Parameters
502  ----------
503  root : string
504  URI to the the root location of the storage
505 
506  Returns
507  -------
508  bool
509  True if the storage exists, false if not
510  """
511  return os.path.exists(PosixStorage._pathFromURI(uri))
512 
513 
514 def readConfigStorage(butlerLocation):
515  """Read from a butlerLocation.
516 
517  Parameters
518  ----------
519  butlerLocation : ButlerLocation
520  The location & formatting for the object(s) to be read.
521 
522  Returns
523  -------
524  A list of objects as described by the butler location. One item for
525  each location in butlerLocation.getLocations()
526  """
527  results = []
528  for locationString in butlerLocation.getLocations():
529  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
530  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
531  if not os.path.exists(logLoc.locString()):
532  raise RuntimeError("No such config file: " + logLoc.locString())
533  pythonType = butlerLocation.getPythonType()
534  if pythonType is not None:
535  if isinstance(pythonType, basestring):
536  pythonType = doImport(pythonType)
537  finalItem = pythonType()
538  finalItem.load(logLoc.locString())
539  results.append(finalItem)
540  return results
541 
542 
543 def writeConfigStorage(butlerLocation, obj):
544  """Writes an object to a location and persistence format specified by
545  ButlerLocation
546 
547  Parameters
548  ----------
549  butlerLocation : ButlerLocation
550  The location & formatting for the object to be written.
551  obj : object instance
552  The object to be written.
553  """
554  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
555  with SafeFilename(filename) as locationString:
556  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
557  obj.save(logLoc.locString())
558 
559 
560 def readFitsStorage(butlerLocation):
561  """Read from a butlerLocation.
562 
563  Parameters
564  ----------
565  butlerLocation : ButlerLocation
566  The location & formatting for the object(s) to be read.
567 
568  Returns
569  -------
570  A list of objects as described by the butler location. One item for
571  each location in butlerLocation.getLocations()
572  """
573  results = []
574  for locationString in butlerLocation.getLocations():
575  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
576  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
577  storageList = StorageList()
578  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
579  logLoc)
580  storageList.append(storage)
581  finalItem = PosixStorage.getPersistence().unsafeRetrieve(
582  butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
583  results.append(finalItem)
584  return results
585 
586 
587 def writeFitsStorage(butlerLocation, obj):
588  """Writes an object to a location and persistence format specified by
589  ButlerLocation
590 
591  Parameters
592  ----------
593  butlerLocation : ButlerLocation
594  The location & formatting for the object to be written.
595  obj : object instance
596  The object to be written.
597  """
598  location = butlerLocation.getLocations()[0]
599  with SafeFilename(os.path.join(butlerLocation.getStorage().root, location)) as locationString:
600  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
601  # Create a list of Storages for the item.
602  storageList = StorageList()
603  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
604  storageList.append(storage)
605  persistence = PosixStorage.getPersistence()
606  if hasattr(obj, '__deref__'):
607  # We have a smart pointer, so dereference it.
608  persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
609  else:
610  persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
611 
612 
613 def readPickleStorage(butlerLocation):
614  """Read from a butlerLocation.
615 
616  Parameters
617  ----------
618  butlerLocation : ButlerLocation
619  The location & formatting for the object(s) to be read.
620 
621  Returns
622  -------
623  A list of objects as described by the butler location. One item for
624  each location in butlerLocation.getLocations()
625  """
626  # Create a list of Storages for the item.
627  results = []
628  additionalData = butlerLocation.getAdditionalData()
629  for locationString in butlerLocation.getLocations():
630  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
631  logLoc = LogicalLocation(locStringWithRoot, additionalData)
632  if not os.path.exists(logLoc.locString()):
633  raise RuntimeError("No such pickle file: " + logLoc.locString())
634  with open(logLoc.locString(), "rb") as infile:
635  # py3: We have to specify encoding since some files were written
636  # by python2, and 'latin1' manages that conversion safely. See:
637  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
638  if sys.version_info.major >= 3:
639  finalItem = pickle.load(infile, encoding="latin1")
640  else:
641  finalItem = pickle.load(infile)
642  results.append(finalItem)
643  return results
644 
645 
646 def writePickleStorage(butlerLocation, obj):
647  """Writes an object to a location and persistence format specified by
648  ButlerLocation
649 
650  Parameters
651  ----------
652  butlerLocation : ButlerLocation
653  The location & formatting for the object to be written.
654  obj : object instance
655  The object to be written.
656  """
657  additionalData = butlerLocation.getAdditionalData()
658  locations = butlerLocation.getLocations()
659  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
660  logLoc = LogicalLocation(locationString, additionalData)
661  with open(logLoc.locString(), "wb") as outfile:
662  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
663 
664 
665 def readFitsCatalogStorage(butlerLocation):
666  """Read from a butlerLocation.
667 
668  Parameters
669  ----------
670  butlerLocation : ButlerLocation
671  The location & formatting for the object(s) to be read.
672 
673  Returns
674  -------
675  A list of objects as described by the butler location. One item for
676  each location in butlerLocation.getLocations()
677  """
678  pythonType = butlerLocation.getPythonType()
679  if pythonType is not None:
680  if isinstance(pythonType, basestring):
681  pythonType = doImport(pythonType)
682  results = []
683  additionalData = butlerLocation.getAdditionalData()
684  for locationString in butlerLocation.getLocations():
685  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
686  logLoc = LogicalLocation(locStringWithRoot, additionalData)
687  if not os.path.exists(logLoc.locString()):
688  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
689  kwds = {}
690  if additionalData.exists("hdu"):
691  kwds["hdu"] = additionalData.getInt("hdu")
692  if additionalData.exists("flags"):
693  kwds["flags"] = additionalData.getInt("flags")
694  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
695  results.append(finalItem)
696  return results
697 
698 
699 def writeFitsCatalogStorage(butlerLocation, obj):
700  """Writes an object to a location and persistence format specified by
701  ButlerLocation
702 
703  Parameters
704  ----------
705  butlerLocation : ButlerLocation
706  The location & formatting for the object to be written.
707  obj : object instance
708  The object to be written.
709  """
710  additionalData = butlerLocation.getAdditionalData()
711  locations = butlerLocation.getLocations()
712  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
713  logLoc = LogicalLocation(locationString, additionalData)
714  if additionalData.exists("flags"):
715  kwds = dict(flags=additionalData.getInt("flags"))
716  else:
717  kwds = {}
718  obj.writeFits(logLoc.locString(), **kwds)
719  return
720 
721 
722 def readPafStorage(butlerLocation):
723  """Read from a butlerLocation.
724 
725  Parameters
726  ----------
727  butlerLocation : ButlerLocation
728  The location & formatting for the object(s) to be read.
729 
730  Returns
731  -------
732  A list of objects as described by the butler location. One item for
733  each location in butlerLocation.getLocations()
734  """
735  results = []
736  for locationString in butlerLocation.getLocations():
737  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
738  butlerLocation.getAdditionalData())
739  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
740  results.append(finalItem)
741  return results
742 
743 
744 def readYamlStorage(butlerLocation):
745  """Read from a butlerLocation.
746 
747  Parameters
748  ----------
749  butlerLocation : ButlerLocation
750  The location & formatting for the object(s) to be read.
751 
752  Returns
753  -------
754  A list of objects as described by the butler location. One item for
755  each location in butlerLocation.getLocations()
756  """
757  results = []
758  for locationString in butlerLocation.getLocations():
759  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
760  butlerLocation.getAdditionalData())
761  finalItem = Policy(filePath=logLoc.locString())
762  results.append(finalItem)
763  return results
764 
765 
766 def readBoostStorage(butlerLocation):
767  results = []
768  additionalData = butlerLocation.getAdditionalData()
769  for locationString in butlerLocation.getLocations():
770  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
771  butlerLocation.getAdditionalData())
772  storageList = StorageList()
773  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
774  storageList.append(storage)
775  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
776  additionalData)
777  results.append(finalItem)
778  return results
779 
780 
781 def writeBoostStorage(butlerLocation, obj):
782  additionalData = butlerLocation.getAdditionalData()
783  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
784  with SafeFilename(location) as locationString:
785  logLoc = LogicalLocation(locationString, additionalData)
786  # Create a list of Storages for the item.
787  storageList = StorageList()
788  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
789  storageList.append(storage)
790  # Persist the item.
791  if hasattr(obj, '__deref__'):
792  # We have a smart pointer, so dereference it.
793  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
794  else:
795  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
796 
797 
798 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
799 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
800 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
801 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
802 PosixStorage.registerFormatters("PafStorage", writeFormatter=readPafStorage)
803 PosixStorage.registerFormatters("YamlStorage", readFormatter=readYamlStorage)
804 PosixStorage.registerFormatters("BoostStorage", readFitsStorage, writeFitsStorage)
805 
806 Storage.registerStorageClass(scheme='', cls=PosixStorage)
807 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)