lsst.daf.persistence  15.0-6-g4cfb9db+1
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import urllib.parse
30 import glob
31 import shutil
32 
33 from . import (LogicalLocation, Persistence, Policy, StorageList,
34  StorageInterface, Storage, ButlerLocation,
35  NoRepositroyAtRoot, RepositoryCfg, doImport)
36 from lsst.log import Log
37 import lsst.pex.policy as pexPolicy
38 from .safeFileIo import SafeFilename, safeMakeDir
39 
40 
41 __all__ = ["PosixStorage"]
42 
43 
45  """Defines the interface for a storage location on the local filesystem.
46 
47  Parameters
48  ----------
49  uri : string
50  URI or path that is used as the storage location.
51  create : bool
52  If True a new repository will be created at the root location if it
53  does not exist. If False then a new repository will not be created.
54 
55  Raises
56  ------
57  NoRepositroyAtRoot
58  If create is False and a repository does not exist at the root
59  specified by uri then NoRepositroyAtRoot is raised.
60  """
61 
62  def __init__(self, uri, create):
63  self.log = Log.getLogger("daf.persistence.butler")
64  self.root = self._pathFromURI(uri)
65  if self.root and not os.path.exists(self.root):
66  if not create:
67  raise NoRepositroyAtRoot("No repository at {}".format(uri))
68  safeMakeDir(self.root)
69 
71 
72  @staticmethod
74  # Always use an empty Persistence policy until we can get rid of it
75  persistencePolicy = pexPolicy.Policy()
76  return Persistence.getPersistence(persistencePolicy)
77 
78  def __repr__(self):
79  return 'PosixStorage(root=%s)' % self.root
80 
81  @staticmethod
82  def _pathFromURI(uri):
83  """Get the path part of the URI"""
84  return urllib.parse.urlparse(uri).path
85 
86  @staticmethod
87  def relativePath(fromPath, toPath):
88  """Get a relative path from a location to a location.
89 
90  Parameters
91  ----------
92  fromPath : string
93  A path at which to start. It can be a relative path or an
94  absolute path.
95  toPath : string
96  A target location. It can be a relative path or an absolute path.
97 
98  Returns
99  -------
100  string
101  A relative path that describes the path from fromPath to toPath.
102  """
103  fromPath = os.path.realpath(fromPath)
104  return os.path.relpath(toPath, fromPath)
105 
106  @staticmethod
107  def absolutePath(fromPath, relativePath):
108  """Get an absolute path for the path from fromUri to toUri
109 
110  Parameters
111  ----------
112  fromPath : the starting location
113  A location at which to start. It can be a relative path or an
114  absolute path.
115  relativePath : the location relative to fromPath
116  A relative path.
117 
118  Returns
119  -------
120  string
121  Path that is an absolute path representation of fromPath +
122  relativePath, if one exists. If relativePath is absolute or if
123  fromPath is not related to relativePath then relativePath will be
124  returned.
125  """
126  if os.path.isabs(relativePath):
127  return relativePath
128  fromPath = os.path.realpath(fromPath)
129  return os.path.normpath(os.path.join(fromPath, relativePath))
130 
131  @staticmethod
133  """Get a persisted RepositoryCfg
134 
135  Parameters
136  ----------
137  uri : URI or path to a RepositoryCfg
138  Description
139 
140  Returns
141  -------
142  A RepositoryCfg instance or None
143  """
144  storage = Storage.makeFromURI(uri)
145  location = ButlerLocation(pythonType=RepositoryCfg,
146  cppType=None,
147  storageName=None,
148  locationList='repositoryCfg.yaml',
149  dataId={},
150  mapper=None,
151  storage=storage,
152  usedDataId=None,
153  datasetType=None)
154  return storage.read(location)
155 
156  @staticmethod
157  def putRepositoryCfg(cfg, loc=None):
158  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
159  location = ButlerLocation(pythonType=RepositoryCfg,
160  cppType=None,
161  storageName=None,
162  locationList='repositoryCfg.yaml',
163  dataId={},
164  mapper=None,
165  storage=storage,
166  usedDataId=None,
167  datasetType=None)
168  storage.write(location, cfg)
169 
170  @staticmethod
171  def getMapperClass(root):
172  """Get the mapper class associated with a repository root.
173 
174  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
175  new code and repositories; they should use the Repository parentCfg mechanism.
176 
177  Parameters
178  ----------
179  root : string
180  The location of a persisted ReositoryCfg is (new style repos), or
181  the location where a _mapper file is (old style repos).
182 
183  Returns
184  -------
185  A class object or a class instance, depending on the state of the
186  mapper when the repository was created.
187  """
188  if not (root):
189  return None
190 
191  cfg = PosixStorage.getRepositoryCfg(root)
192  if cfg is not None:
193  return cfg.mapper
194 
195  # Find a "_mapper" file containing the mapper class name
196  basePath = root
197  mapperFile = "_mapper"
198  while not os.path.exists(os.path.join(basePath, mapperFile)):
199  # Break abstraction by following _parent links from CameraMapper
200  if os.path.exists(os.path.join(basePath, "_parent")):
201  basePath = os.path.join(basePath, "_parent")
202  else:
203  mapperFile = None
204  break
205 
206  if mapperFile is not None:
207  mapperFile = os.path.join(basePath, mapperFile)
208 
209  # Read the name of the mapper class and instantiate it
210  with open(mapperFile, "r") as f:
211  mapperName = f.readline().strip()
212  components = mapperName.split(".")
213  if len(components) <= 1:
214  raise RuntimeError("Unqualified mapper name %s in %s" %
215  (mapperName, mapperFile))
216  pkg = importlib.import_module(".".join(components[:-1]))
217  return getattr(pkg, components[-1])
218 
219  return None
220 
221  @staticmethod
223  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
224  symlink.
225 
226  Parameters
227  ----------
228  root : string
229  A path to the folder on the local filesystem.
230 
231  Returns
232  -------
233  string or None
234  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
235  symlink at root.
236  """
237  linkpath = os.path.join(root, '_parent')
238  if os.path.exists(linkpath):
239  try:
240  return os.readlink(os.path.join(root, '_parent'))
241  except OSError:
242  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
243  # location. Allow that; return the path of that folder.
244  return os.path.join(root, '_parent')
245  return None
246 
247  def write(self, butlerLocation, obj):
248  """Writes an object to a location and persistence format specified by
249  ButlerLocation
250 
251  Parameters
252  ----------
253  butlerLocation : ButlerLocation
254  The location & formatting for the object to be written.
255  obj : object instance
256  The object to be written.
257  """
258  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
259 
260  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
261  if not writeFormatter:
262  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
263  if writeFormatter:
264  writeFormatter(butlerLocation, obj)
265  return
266 
267  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
268 
269  def read(self, butlerLocation):
270  """Read from a butlerLocation.
271 
272  Parameters
273  ----------
274  butlerLocation : ButlerLocation
275  The location & formatting for the object(s) to be read.
276 
277  Returns
278  -------
279  A list of objects as described by the butler location. One item for
280  each location in butlerLocation.getLocations()
281  """
282  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
283  if not readFormatter:
284  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
285  if readFormatter:
286  return readFormatter(butlerLocation)
287 
288  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
289 
290  def butlerLocationExists(self, location):
291  """Implementation of PosixStorage.exists for ButlerLocation objects.
292  """
293  storageName = location.getStorageName()
294  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
295  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
296  'ParquetStorage'):
297  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
298  return False
299  for locationString in location.getLocations():
300  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
301  obj = self.instanceSearch(path=logLoc)
302  if obj:
303  return True
304  return False
305 
306  def exists(self, location):
307  """Check if location exists.
308 
309  Parameters
310  ----------
311  location : ButlerLocation or string
312  A a string or a ButlerLocation that describes the location of an
313  object in this storage.
314 
315  Returns
316  -------
317  bool
318  True if exists, else False.
319  """
320  if isinstance(location, ButlerLocation):
321  return self.butlerLocationExists(location)
322 
323  obj = self.instanceSearch(path=location)
324  return bool(obj)
325 
326  def locationWithRoot(self, location):
327  """Get the full path to the location.
328 
329  :param location:
330  :return:
331  """
332  return os.path.join(self.root, location)
333 
334  @staticmethod
335  def v1RepoExists(root):
336  """Test if a Version 1 Repository exists.
337 
338  Version 1 Repositories only exist in posix storages, do not have a
339  RepositoryCfg file, and contain either a registry.sqlite3 file, a
340  _mapper file, or a _parent link.
341 
342  Parameters
343  ----------
344  root : string
345  A path to a folder on the local filesystem.
346 
347  Returns
348  -------
349  bool
350  True if the repository at root exists, else False.
351  """
352  return os.path.exists(root) and (
353  os.path.exists(os.path.join(root, "registry.sqlite3")) or
354  os.path.exists(os.path.join(root, "_mapper")) or
355  os.path.exists(os.path.join(root, "_parent"))
356  )
357 
358  def copyFile(self, fromLocation, toLocation):
359  """Copy a file from one location to another on the local filesystem.
360 
361  Parameters
362  ----------
363  fromLocation : path
364  Path and name of existing file.
365  toLocation : path
366  Path and name of new file.
367 
368  Returns
369  -------
370  None
371  """
372  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
373 
374  def getLocalFile(self, path):
375  """Get a handle to a local copy of the file, downloading it to a
376  temporary if needed.
377 
378  Parameters
379  ----------
380  A path the the file in storage, relative to root.
381 
382  Returns
383  -------
384  A handle to a local copy of the file. If storage is remote it will be
385  a temporary file. If storage is local it may be the original file or
386  a temporary file. The file name can be gotten via the 'name' property
387  of the returned object.
388  """
389  p = os.path.join(self.root, path)
390  try:
391  return open(p)
392  except IOError as e:
393  if e.errno == 2: # 'No such file or directory'
394  return None
395  else:
396  raise e
397 
398  def instanceSearch(self, path):
399  """Search for the given path in this storage instance.
400 
401  If the path contains an HDU indicator (a number in brackets before the
402  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
403  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
404  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
405 
406  Parameters
407  ----------
408  path : string
409  A filename (and optionally prefix path) to search for within root.
410 
411  Returns
412  -------
413  string or None
414  The location that was found, or None if no location was found.
415  """
416  return self.search(self.root, path)
417 
418  @staticmethod
419  def search(root, path, searchParents=False):
420  """Look for the given path in the current root.
421 
422  Also supports searching for the path in Butler v1 repositories by
423  following the Butler v1 _parent symlink
424 
425  If the path contains an HDU indicator (a number in brackets, e.g.
426  'foo.fits[1]', this will be stripped when searching and so
427  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
428  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
429 
430  Parameters
431  ----------
432  root : string
433  The path to the root directory.
434  path : string
435  The path to the file within the root directory.
436  searchParents : bool, optional
437  For Butler v1 repositories only, if true and a _parent symlink
438  exists, then the directory at _parent will be searched if the file
439  is not found in the root repository. Will continue searching the
440  parent of the parent until the file is found or no additional
441  parent exists.
442 
443  Returns
444  -------
445  string or None
446  The location that was found, or None if no location was found.
447  """
448  # Separate path into a root-equivalent prefix (in dir) and the rest
449  # (left in path)
450  rootDir = root
451  # First remove trailing slashes (#2527)
452  while len(rootDir) > 1 and rootDir[-1] == '/':
453  rootDir = rootDir[:-1]
454 
455  if path.startswith(rootDir + "/"):
456  # Common case; we have the same root prefix string
457  path = path[len(rootDir + '/'):]
458  pathPrefix = rootDir
459  elif rootDir == "/" and path.startswith("/"):
460  path = path[1:]
461  pathPrefix = None
462  else:
463  # Search for prefix that is the same as root
464  pathPrefix = os.path.dirname(path)
465  while pathPrefix != "" and pathPrefix != "/":
466  if os.path.realpath(pathPrefix) == os.path.realpath(root):
467  break
468  pathPrefix = os.path.dirname(pathPrefix)
469  if pathPrefix == "/":
470  path = path[1:]
471  elif pathPrefix != "":
472  path = path[len(pathPrefix)+1:]
473 
474  # Now search for the path in the root or its parents
475  # Strip off any cfitsio bracketed extension if present
476  strippedPath = path
477  pathStripped = None
478  firstBracket = path.find("[")
479  if firstBracket != -1:
480  strippedPath = path[:firstBracket]
481  pathStripped = path[firstBracket:]
482 
483  dir = rootDir
484  while True:
485  paths = glob.glob(os.path.join(dir, strippedPath))
486  if len(paths) > 0:
487  if pathPrefix != rootDir:
488  paths = [p[len(rootDir+'/'):] for p in paths]
489  if pathStripped is not None:
490  paths = [p + pathStripped for p in paths]
491  return paths
492  if searchParents:
493  dir = os.path.join(dir, "_parent")
494  if not os.path.exists(dir):
495  return None
496  else:
497  return None
498 
499  @staticmethod
500  def storageExists(uri):
501  """Ask if a storage at the location described by uri exists
502 
503  Parameters
504  ----------
505  root : string
506  URI to the the root location of the storage
507 
508  Returns
509  -------
510  bool
511  True if the storage exists, false if not
512  """
513  return os.path.exists(PosixStorage._pathFromURI(uri))
514 
515 
516 def readConfigStorage(butlerLocation):
517  """Read from a butlerLocation.
518 
519  Parameters
520  ----------
521  butlerLocation : ButlerLocation
522  The location & formatting for the object(s) to be read.
523 
524  Returns
525  -------
526  A list of objects as described by the butler location. One item for
527  each location in butlerLocation.getLocations()
528  """
529  results = []
530  for locationString in butlerLocation.getLocations():
531  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
532  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
533  if not os.path.exists(logLoc.locString()):
534  raise RuntimeError("No such config file: " + logLoc.locString())
535  pythonType = butlerLocation.getPythonType()
536  if pythonType is not None:
537  if isinstance(pythonType, basestring):
538  pythonType = doImport(pythonType)
539  finalItem = pythonType()
540  finalItem.load(logLoc.locString())
541  results.append(finalItem)
542  return results
543 
544 
545 def writeConfigStorage(butlerLocation, obj):
546  """Writes an object to a location and persistence format specified by
547  ButlerLocation
548 
549  Parameters
550  ----------
551  butlerLocation : ButlerLocation
552  The location & formatting for the object to be written.
553  obj : object instance
554  The object to be written.
555  """
556  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
557  with SafeFilename(filename) as locationString:
558  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
559  obj.save(logLoc.locString())
560 
561 
562 def readFitsStorage(butlerLocation):
563  """Read from a butlerLocation.
564 
565  Parameters
566  ----------
567  butlerLocation : ButlerLocation
568  The location & formatting for the object(s) to be read.
569 
570  Returns
571  -------
572  A list of objects as described by the butler location. One item for
573  each location in butlerLocation.getLocations()
574  """
575  results = []
576  for locationString in butlerLocation.getLocations():
577  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
578  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
579  storageList = StorageList()
580  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
581  logLoc)
582  storageList.append(storage)
583  finalItem = PosixStorage.getPersistence().unsafeRetrieve(
584  butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
585  results.append(finalItem)
586  return results
587 
588 
589 def writeFitsStorage(butlerLocation, obj):
590  """Writes an object to a location and persistence format specified by
591  ButlerLocation
592 
593  Parameters
594  ----------
595  butlerLocation : ButlerLocation
596  The location & formatting for the object to be written.
597  obj : object instance
598  The object to be written.
599  """
600  location = butlerLocation.getLocations()[0]
601  with SafeFilename(os.path.join(butlerLocation.getStorage().root, location)) as locationString:
602  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
603  # Create a list of Storages for the item.
604  storageList = StorageList()
605  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
606  storageList.append(storage)
607  persistence = PosixStorage.getPersistence()
608  if hasattr(obj, '__deref__'):
609  # We have a smart pointer, so dereference it.
610  persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
611  else:
612  persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
613 
614 
615 def readParquetStorage(butlerLocation):
616  """Read from a butlerLocation.
617 
618  The object returned by this is expected to be a subtype
619  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
620  that allows for lazy loading of the data.
621 
622  Parameters
623  ----------
624  butlerLocation : ButlerLocation
625  The location & formatting for the object(s) to be read.
626 
627  Returns
628  -------
629  A list of objects as described by the butler location. One item for
630  each location in butlerLocation.getLocations()
631  """
632  results = []
633  additionalData = butlerLocation.getAdditionalData()
634 
635  for locationString in butlerLocation.getLocations():
636  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
637  logLoc = LogicalLocation(locStringWithRoot, additionalData)
638  if not os.path.exists(logLoc.locString()):
639  raise RuntimeError("No such parquet file: " + logLoc.locString())
640 
641  pythonType = butlerLocation.getPythonType()
642  if pythonType is not None:
643  if isinstance(pythonType, basestring):
644  pythonType = doImport(pythonType)
645 
646  filename = logLoc.locString()
647 
648  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
649  # filename should be the first kwarg, but being explicit here.
650  results.append(pythonType(filename=filename))
651 
652  if len(results) > 1:
653  Log.getLogger("daf.persistence.butler").warning('Not using multiple locations!')
654 
655  return results[0]
656 
657 
658 def writeParquetStorage(butlerLocation, obj):
659  """Writes pandas dataframe to parquet file
660 
661  Parameters
662  ----------
663  butlerLocation : ButlerLocation
664  The location & formatting for the object(s) to be read.
665  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
666  Wrapped DataFrame to write.
667 
668  """
669  additionalData = butlerLocation.getAdditionalData()
670  locations = butlerLocation.getLocations()
671  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
672  logLoc = LogicalLocation(locationString, additionalData)
673  filename = logLoc.locString()
674  obj.write(filename)
675 
676 
677 def readPickleStorage(butlerLocation):
678  """Read from a butlerLocation.
679 
680  Parameters
681  ----------
682  butlerLocation : ButlerLocation
683  The location & formatting for the object(s) to be read.
684 
685  Returns
686  -------
687  A list of objects as described by the butler location. One item for
688  each location in butlerLocation.getLocations()
689  """
690  # Create a list of Storages for the item.
691  results = []
692  additionalData = butlerLocation.getAdditionalData()
693  for locationString in butlerLocation.getLocations():
694  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
695  logLoc = LogicalLocation(locStringWithRoot, additionalData)
696  if not os.path.exists(logLoc.locString()):
697  raise RuntimeError("No such pickle file: " + logLoc.locString())
698  with open(logLoc.locString(), "rb") as infile:
699  # py3: We have to specify encoding since some files were written
700  # by python2, and 'latin1' manages that conversion safely. See:
701  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
702  if sys.version_info.major >= 3:
703  finalItem = pickle.load(infile, encoding="latin1")
704  else:
705  finalItem = pickle.load(infile)
706  results.append(finalItem)
707  return results
708 
709 
710 def writePickleStorage(butlerLocation, obj):
711  """Writes an object to a location and persistence format specified by
712  ButlerLocation
713 
714  Parameters
715  ----------
716  butlerLocation : ButlerLocation
717  The location & formatting for the object to be written.
718  obj : object instance
719  The object to be written.
720  """
721  additionalData = butlerLocation.getAdditionalData()
722  locations = butlerLocation.getLocations()
723  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
724  logLoc = LogicalLocation(locationString, additionalData)
725  with open(logLoc.locString(), "wb") as outfile:
726  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
727 
728 
729 def readFitsCatalogStorage(butlerLocation):
730  """Read from a butlerLocation.
731 
732  Parameters
733  ----------
734  butlerLocation : ButlerLocation
735  The location & formatting for the object(s) to be read.
736 
737  Returns
738  -------
739  A list of objects as described by the butler location. One item for
740  each location in butlerLocation.getLocations()
741  """
742  pythonType = butlerLocation.getPythonType()
743  if pythonType is not None:
744  if isinstance(pythonType, basestring):
745  pythonType = doImport(pythonType)
746  results = []
747  additionalData = butlerLocation.getAdditionalData()
748  for locationString in butlerLocation.getLocations():
749  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
750  logLoc = LogicalLocation(locStringWithRoot, additionalData)
751  if not os.path.exists(logLoc.locString()):
752  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
753  kwds = {}
754  if additionalData.exists("hdu"):
755  kwds["hdu"] = additionalData.getInt("hdu")
756  if additionalData.exists("flags"):
757  kwds["flags"] = additionalData.getInt("flags")
758  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
759  results.append(finalItem)
760  return results
761 
762 
763 def writeFitsCatalogStorage(butlerLocation, obj):
764  """Writes an object to a location and persistence format specified by
765  ButlerLocation
766 
767  Parameters
768  ----------
769  butlerLocation : ButlerLocation
770  The location & formatting for the object to be written.
771  obj : object instance
772  The object to be written.
773  """
774  additionalData = butlerLocation.getAdditionalData()
775  locations = butlerLocation.getLocations()
776  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
777  logLoc = LogicalLocation(locationString, additionalData)
778  if additionalData.exists("flags"):
779  kwds = dict(flags=additionalData.getInt("flags"))
780  else:
781  kwds = {}
782  obj.writeFits(logLoc.locString(), **kwds)
783  return
784 
785 
786 def readPafStorage(butlerLocation):
787  """Read from a butlerLocation.
788 
789  Parameters
790  ----------
791  butlerLocation : ButlerLocation
792  The location & formatting for the object(s) to be read.
793 
794  Returns
795  -------
796  A list of objects as described by the butler location. One item for
797  each location in butlerLocation.getLocations()
798  """
799  results = []
800  for locationString in butlerLocation.getLocations():
801  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
802  butlerLocation.getAdditionalData())
803  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
804  results.append(finalItem)
805  return results
806 
807 
808 def readYamlStorage(butlerLocation):
809  """Read from a butlerLocation.
810 
811  Parameters
812  ----------
813  butlerLocation : ButlerLocation
814  The location & formatting for the object(s) to be read.
815 
816  Returns
817  -------
818  A list of objects as described by the butler location. One item for
819  each location in butlerLocation.getLocations()
820  """
821  results = []
822  for locationString in butlerLocation.getLocations():
823  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
824  butlerLocation.getAdditionalData())
825  finalItem = Policy(filePath=logLoc.locString())
826  results.append(finalItem)
827  return results
828 
829 
830 def readBoostStorage(butlerLocation):
831  results = []
832  additionalData = butlerLocation.getAdditionalData()
833  for locationString in butlerLocation.getLocations():
834  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
835  butlerLocation.getAdditionalData())
836  storageList = StorageList()
837  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
838  storageList.append(storage)
839  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
840  additionalData)
841  results.append(finalItem)
842  return results
843 
844 
845 def writeBoostStorage(butlerLocation, obj):
846  additionalData = butlerLocation.getAdditionalData()
847  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
848  with SafeFilename(location) as locationString:
849  logLoc = LogicalLocation(locationString, additionalData)
850  # Create a list of Storages for the item.
851  storageList = StorageList()
852  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
853  storageList.append(storage)
854  # Persist the item.
855  if hasattr(obj, '__deref__'):
856  # We have a smart pointer, so dereference it.
857  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
858  else:
859  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
860 
861 
862 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
863 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
864 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
865 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
866 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
867 PosixStorage.registerFormatters("PafStorage", writeFormatter=readPafStorage)
868 PosixStorage.registerFormatters("YamlStorage", readFormatter=readYamlStorage)
869 PosixStorage.registerFormatters("BoostStorage", readFitsStorage, writeFitsStorage)
870 
871 Storage.registerStorageClass(scheme='', cls=PosixStorage)
872 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)