lsst.daf.persistence  15.0-9-g1e7c341+6
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import urllib.parse
30 import glob
31 import shutil
32 
33 from . import (LogicalLocation, Persistence, Policy, StorageList,
34  StorageInterface, Storage, ButlerLocation,
35  NoRepositroyAtRoot, RepositoryCfg, doImport)
36 from lsst.log import Log
37 import lsst.pex.policy as pexPolicy
38 from .safeFileIo import SafeFilename, safeMakeDir
39 
40 
41 __all__ = ["PosixStorage"]
42 
43 
45  """Defines the interface for a storage location on the local filesystem.
46 
47  Parameters
48  ----------
49  uri : string
50  URI or path that is used as the storage location.
51  create : bool
52  If True a new repository will be created at the root location if it
53  does not exist. If False then a new repository will not be created.
54 
55  Raises
56  ------
57  NoRepositroyAtRoot
58  If create is False and a repository does not exist at the root
59  specified by uri then NoRepositroyAtRoot is raised.
60  """
61 
62  def __init__(self, uri, create):
63  self.log = Log.getLogger("daf.persistence.butler")
64  self.root = self._pathFromURI(uri)
65  if self.root and not os.path.exists(self.root):
66  if not create:
67  raise NoRepositroyAtRoot("No repository at {}".format(uri))
68  safeMakeDir(self.root)
69 
71 
72  @staticmethod
74  # Always use an empty Persistence policy until we can get rid of it
75  persistencePolicy = pexPolicy.Policy()
76  return Persistence.getPersistence(persistencePolicy)
77 
78  def __repr__(self):
79  return 'PosixStorage(root=%s)' % self.root
80 
81  @staticmethod
82  def _pathFromURI(uri):
83  """Get the path part of the URI"""
84  return urllib.parse.urlparse(uri).path
85 
86  @staticmethod
87  def relativePath(fromPath, toPath):
88  """Get a relative path from a location to a location.
89 
90  Parameters
91  ----------
92  fromPath : string
93  A path at which to start. It can be a relative path or an
94  absolute path.
95  toPath : string
96  A target location. It can be a relative path or an absolute path.
97 
98  Returns
99  -------
100  string
101  A relative path that describes the path from fromPath to toPath.
102  """
103  fromPath = os.path.realpath(fromPath)
104  return os.path.relpath(toPath, fromPath)
105 
106  @staticmethod
107  def absolutePath(fromPath, relativePath):
108  """Get an absolute path for the path from fromUri to toUri
109 
110  Parameters
111  ----------
112  fromPath : the starting location
113  A location at which to start. It can be a relative path or an
114  absolute path.
115  relativePath : the location relative to fromPath
116  A relative path.
117 
118  Returns
119  -------
120  string
121  Path that is an absolute path representation of fromPath +
122  relativePath, if one exists. If relativePath is absolute or if
123  fromPath is not related to relativePath then relativePath will be
124  returned.
125  """
126  if os.path.isabs(relativePath):
127  return relativePath
128  fromPath = os.path.realpath(fromPath)
129  return os.path.normpath(os.path.join(fromPath, relativePath))
130 
131  @staticmethod
133  """Get a persisted RepositoryCfg
134 
135  Parameters
136  ----------
137  uri : URI or path to a RepositoryCfg
138  Description
139 
140  Returns
141  -------
142  A RepositoryCfg instance or None
143  """
144  storage = Storage.makeFromURI(uri)
145  location = ButlerLocation(pythonType=RepositoryCfg,
146  cppType=None,
147  storageName=None,
148  locationList='repositoryCfg.yaml',
149  dataId={},
150  mapper=None,
151  storage=storage,
152  usedDataId=None,
153  datasetType=None)
154  return storage.read(location)
155 
156  @staticmethod
157  def putRepositoryCfg(cfg, loc=None):
158  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
159  location = ButlerLocation(pythonType=RepositoryCfg,
160  cppType=None,
161  storageName=None,
162  locationList='repositoryCfg.yaml',
163  dataId={},
164  mapper=None,
165  storage=storage,
166  usedDataId=None,
167  datasetType=None)
168  storage.write(location, cfg)
169 
170  @staticmethod
171  def getMapperClass(root):
172  """Get the mapper class associated with a repository root.
173 
174  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
175  new code and repositories; they should use the Repository parentCfg mechanism.
176 
177  Parameters
178  ----------
179  root : string
180  The location of a persisted ReositoryCfg is (new style repos), or
181  the location where a _mapper file is (old style repos).
182 
183  Returns
184  -------
185  A class object or a class instance, depending on the state of the
186  mapper when the repository was created.
187  """
188  if not (root):
189  return None
190 
191  cfg = PosixStorage.getRepositoryCfg(root)
192  if cfg is not None:
193  return cfg.mapper
194 
195  # Find a "_mapper" file containing the mapper class name
196  basePath = root
197  mapperFile = "_mapper"
198  while not os.path.exists(os.path.join(basePath, mapperFile)):
199  # Break abstraction by following _parent links from CameraMapper
200  if os.path.exists(os.path.join(basePath, "_parent")):
201  basePath = os.path.join(basePath, "_parent")
202  else:
203  mapperFile = None
204  break
205 
206  if mapperFile is not None:
207  mapperFile = os.path.join(basePath, mapperFile)
208 
209  # Read the name of the mapper class and instantiate it
210  with open(mapperFile, "r") as f:
211  mapperName = f.readline().strip()
212  components = mapperName.split(".")
213  if len(components) <= 1:
214  raise RuntimeError("Unqualified mapper name %s in %s" %
215  (mapperName, mapperFile))
216  pkg = importlib.import_module(".".join(components[:-1]))
217  return getattr(pkg, components[-1])
218 
219  return None
220 
221  @staticmethod
223  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
224  symlink.
225 
226  Parameters
227  ----------
228  root : string
229  A path to the folder on the local filesystem.
230 
231  Returns
232  -------
233  string or None
234  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
235  symlink at root.
236  """
237  linkpath = os.path.join(root, '_parent')
238  if os.path.exists(linkpath):
239  try:
240  return os.readlink(os.path.join(root, '_parent'))
241  except OSError:
242  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
243  # location. Allow that; return the path of that folder.
244  return os.path.join(root, '_parent')
245  return None
246 
247  def write(self, butlerLocation, obj):
248  """Writes an object to a location and persistence format specified by
249  ButlerLocation
250 
251  Parameters
252  ----------
253  butlerLocation : ButlerLocation
254  The location & formatting for the object to be written.
255  obj : object instance
256  The object to be written.
257  """
258  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
259 
260  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
261  if not writeFormatter:
262  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
263  if writeFormatter:
264  writeFormatter(butlerLocation, obj)
265  return
266 
267  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
268 
269  def read(self, butlerLocation):
270  """Read from a butlerLocation.
271 
272  Parameters
273  ----------
274  butlerLocation : ButlerLocation
275  The location & formatting for the object(s) to be read.
276 
277  Returns
278  -------
279  A list of objects as described by the butler location. One item for
280  each location in butlerLocation.getLocations()
281  """
282  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
283  if not readFormatter:
284  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
285  if readFormatter:
286  return readFormatter(butlerLocation)
287 
288  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
289 
290  def butlerLocationExists(self, location):
291  """Implementation of PosixStorage.exists for ButlerLocation objects.
292  """
293  storageName = location.getStorageName()
294  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
295  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
296  'ParquetStorage', 'MatplotlibStorage'):
297  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
298  return False
299  for locationString in location.getLocations():
300  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
301  obj = self.instanceSearch(path=logLoc)
302  if obj:
303  return True
304  return False
305 
306  def exists(self, location):
307  """Check if location exists.
308 
309  Parameters
310  ----------
311  location : ButlerLocation or string
312  A a string or a ButlerLocation that describes the location of an
313  object in this storage.
314 
315  Returns
316  -------
317  bool
318  True if exists, else False.
319  """
320  if isinstance(location, ButlerLocation):
321  return self.butlerLocationExists(location)
322 
323  obj = self.instanceSearch(path=location)
324  return bool(obj)
325 
326  def locationWithRoot(self, location):
327  """Get the full path to the location.
328 
329  :param location:
330  :return:
331  """
332  return os.path.join(self.root, location)
333 
334  @staticmethod
335  def v1RepoExists(root):
336  """Test if a Version 1 Repository exists.
337 
338  Version 1 Repositories only exist in posix storages, do not have a
339  RepositoryCfg file, and contain either a registry.sqlite3 file, a
340  _mapper file, or a _parent link.
341 
342  Parameters
343  ----------
344  root : string
345  A path to a folder on the local filesystem.
346 
347  Returns
348  -------
349  bool
350  True if the repository at root exists, else False.
351  """
352  return os.path.exists(root) and (
353  os.path.exists(os.path.join(root, "registry.sqlite3")) or
354  os.path.exists(os.path.join(root, "_mapper")) or
355  os.path.exists(os.path.join(root, "_parent"))
356  )
357 
358  def copyFile(self, fromLocation, toLocation):
359  """Copy a file from one location to another on the local filesystem.
360 
361  Parameters
362  ----------
363  fromLocation : path
364  Path and name of existing file.
365  toLocation : path
366  Path and name of new file.
367 
368  Returns
369  -------
370  None
371  """
372  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
373 
374  def getLocalFile(self, path):
375  """Get a handle to a local copy of the file, downloading it to a
376  temporary if needed.
377 
378  Parameters
379  ----------
380  A path the the file in storage, relative to root.
381 
382  Returns
383  -------
384  A handle to a local copy of the file. If storage is remote it will be
385  a temporary file. If storage is local it may be the original file or
386  a temporary file. The file name can be gotten via the 'name' property
387  of the returned object.
388  """
389  p = os.path.join(self.root, path)
390  try:
391  return open(p)
392  except IOError as e:
393  if e.errno == 2: # 'No such file or directory'
394  return None
395  else:
396  raise e
397 
398  def instanceSearch(self, path):
399  """Search for the given path in this storage instance.
400 
401  If the path contains an HDU indicator (a number in brackets before the
402  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
403  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
404  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
405 
406  Parameters
407  ----------
408  path : string
409  A filename (and optionally prefix path) to search for within root.
410 
411  Returns
412  -------
413  string or None
414  The location that was found, or None if no location was found.
415  """
416  return self.search(self.root, path)
417 
418  @staticmethod
419  def search(root, path, searchParents=False):
420  """Look for the given path in the current root.
421 
422  Also supports searching for the path in Butler v1 repositories by
423  following the Butler v1 _parent symlink
424 
425  If the path contains an HDU indicator (a number in brackets, e.g.
426  'foo.fits[1]', this will be stripped when searching and so
427  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
428  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
429 
430  Parameters
431  ----------
432  root : string
433  The path to the root directory.
434  path : string
435  The path to the file within the root directory.
436  searchParents : bool, optional
437  For Butler v1 repositories only, if true and a _parent symlink
438  exists, then the directory at _parent will be searched if the file
439  is not found in the root repository. Will continue searching the
440  parent of the parent until the file is found or no additional
441  parent exists.
442 
443  Returns
444  -------
445  string or None
446  The location that was found, or None if no location was found.
447  """
448  # Separate path into a root-equivalent prefix (in dir) and the rest
449  # (left in path)
450  rootDir = root
451  # First remove trailing slashes (#2527)
452  while len(rootDir) > 1 and rootDir[-1] == '/':
453  rootDir = rootDir[:-1]
454 
455  if path.startswith(rootDir + "/"):
456  # Common case; we have the same root prefix string
457  path = path[len(rootDir + '/'):]
458  pathPrefix = rootDir
459  elif rootDir == "/" and path.startswith("/"):
460  path = path[1:]
461  pathPrefix = None
462  else:
463  # Search for prefix that is the same as root
464  pathPrefix = os.path.dirname(path)
465  while pathPrefix != "" and pathPrefix != "/":
466  if os.path.realpath(pathPrefix) == os.path.realpath(root):
467  break
468  pathPrefix = os.path.dirname(pathPrefix)
469  if pathPrefix == "/":
470  path = path[1:]
471  elif pathPrefix != "":
472  path = path[len(pathPrefix)+1:]
473 
474  # Now search for the path in the root or its parents
475  # Strip off any cfitsio bracketed extension if present
476  strippedPath = path
477  pathStripped = None
478  firstBracket = path.find("[")
479  if firstBracket != -1:
480  strippedPath = path[:firstBracket]
481  pathStripped = path[firstBracket:]
482 
483  dir = rootDir
484  while True:
485  paths = glob.glob(os.path.join(dir, strippedPath))
486  if len(paths) > 0:
487  if pathPrefix != rootDir:
488  paths = [p[len(rootDir+'/'):] for p in paths]
489  if pathStripped is not None:
490  paths = [p + pathStripped for p in paths]
491  return paths
492  if searchParents:
493  dir = os.path.join(dir, "_parent")
494  if not os.path.exists(dir):
495  return None
496  else:
497  return None
498 
499  @staticmethod
500  def storageExists(uri):
501  """Ask if a storage at the location described by uri exists
502 
503  Parameters
504  ----------
505  root : string
506  URI to the the root location of the storage
507 
508  Returns
509  -------
510  bool
511  True if the storage exists, false if not
512  """
513  return os.path.exists(PosixStorage._pathFromURI(uri))
514 
515 
516 def readConfigStorage(butlerLocation):
517  """Read from a butlerLocation.
518 
519  Parameters
520  ----------
521  butlerLocation : ButlerLocation
522  The location & formatting for the object(s) to be read.
523 
524  Returns
525  -------
526  A list of objects as described by the butler location. One item for
527  each location in butlerLocation.getLocations()
528  """
529  results = []
530  for locationString in butlerLocation.getLocations():
531  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
532  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
533  if not os.path.exists(logLoc.locString()):
534  raise RuntimeError("No such config file: " + logLoc.locString())
535  pythonType = butlerLocation.getPythonType()
536  if pythonType is not None:
537  if isinstance(pythonType, basestring):
538  pythonType = doImport(pythonType)
539  finalItem = pythonType()
540  finalItem.load(logLoc.locString())
541  results.append(finalItem)
542  return results
543 
544 
545 def writeConfigStorage(butlerLocation, obj):
546  """Writes an object to a location and persistence format specified by
547  ButlerLocation
548 
549  Parameters
550  ----------
551  butlerLocation : ButlerLocation
552  The location & formatting for the object to be written.
553  obj : object instance
554  The object to be written.
555  """
556  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
557  with SafeFilename(filename) as locationString:
558  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
559  obj.save(logLoc.locString())
560 
561 
562 def readFitsStorage(butlerLocation):
563  """Read from a butlerLocation.
564 
565  Parameters
566  ----------
567  butlerLocation : ButlerLocation
568  The location & formatting for the object(s) to be read.
569 
570  Returns
571  -------
572  A list of objects as described by the butler location. One item for
573  each location in butlerLocation.getLocations()
574  """
575  results = []
576  for locationString in butlerLocation.getLocations():
577  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
578  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
579  storageList = StorageList()
580  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
581  logLoc)
582  storageList.append(storage)
583  finalItem = PosixStorage.getPersistence().unsafeRetrieve(
584  butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
585  results.append(finalItem)
586  return results
587 
588 
589 def writeFitsStorage(butlerLocation, obj):
590  """Writes an object to a location and persistence format specified by
591  ButlerLocation
592 
593  Parameters
594  ----------
595  butlerLocation : ButlerLocation
596  The location & formatting for the object to be written.
597  obj : object instance
598  The object to be written.
599  """
600  location = butlerLocation.getLocations()[0]
601  with SafeFilename(os.path.join(butlerLocation.getStorage().root, location)) as locationString:
602  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
603  # Create a list of Storages for the item.
604  storageList = StorageList()
605  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
606  storageList.append(storage)
607  persistence = PosixStorage.getPersistence()
608  if hasattr(obj, '__deref__'):
609  # We have a smart pointer, so dereference it.
610  persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
611  else:
612  persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
613 
614 
615 def readParquetStorage(butlerLocation):
616  """Read from a butlerLocation.
617 
618  The object returned by this is expected to be a subtype
619  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
620  that allows for lazy loading of the data.
621 
622  Parameters
623  ----------
624  butlerLocation : ButlerLocation
625  The location & formatting for the object(s) to be read.
626 
627  Returns
628  -------
629  A list of objects as described by the butler location. One item for
630  each location in butlerLocation.getLocations()
631  """
632  results = []
633  additionalData = butlerLocation.getAdditionalData()
634 
635  for locationString in butlerLocation.getLocations():
636  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
637  logLoc = LogicalLocation(locStringWithRoot, additionalData)
638  if not os.path.exists(logLoc.locString()):
639  raise RuntimeError("No such parquet file: " + logLoc.locString())
640 
641  pythonType = butlerLocation.getPythonType()
642  if pythonType is not None:
643  if isinstance(pythonType, basestring):
644  pythonType = doImport(pythonType)
645 
646  filename = logLoc.locString()
647 
648  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
649  # filename should be the first kwarg, but being explicit here.
650  results.append(pythonType(filename=filename))
651 
652  return results
653 
654 
655 def writeParquetStorage(butlerLocation, obj):
656  """Writes pandas dataframe to parquet file
657 
658  Parameters
659  ----------
660  butlerLocation : ButlerLocation
661  The location & formatting for the object(s) to be read.
662  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
663  Wrapped DataFrame to write.
664 
665  """
666  additionalData = butlerLocation.getAdditionalData()
667  locations = butlerLocation.getLocations()
668  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
669  logLoc = LogicalLocation(locationString, additionalData)
670  filename = logLoc.locString()
671  obj.write(filename)
672 
673 
674 def readPickleStorage(butlerLocation):
675  """Read from a butlerLocation.
676 
677  Parameters
678  ----------
679  butlerLocation : ButlerLocation
680  The location & formatting for the object(s) to be read.
681 
682  Returns
683  -------
684  A list of objects as described by the butler location. One item for
685  each location in butlerLocation.getLocations()
686  """
687  # Create a list of Storages for the item.
688  results = []
689  additionalData = butlerLocation.getAdditionalData()
690  for locationString in butlerLocation.getLocations():
691  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
692  logLoc = LogicalLocation(locStringWithRoot, additionalData)
693  if not os.path.exists(logLoc.locString()):
694  raise RuntimeError("No such pickle file: " + logLoc.locString())
695  with open(logLoc.locString(), "rb") as infile:
696  # py3: We have to specify encoding since some files were written
697  # by python2, and 'latin1' manages that conversion safely. See:
698  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
699  if sys.version_info.major >= 3:
700  finalItem = pickle.load(infile, encoding="latin1")
701  else:
702  finalItem = pickle.load(infile)
703  results.append(finalItem)
704  return results
705 
706 
707 def writePickleStorage(butlerLocation, obj):
708  """Writes an object to a location and persistence format specified by
709  ButlerLocation
710 
711  Parameters
712  ----------
713  butlerLocation : ButlerLocation
714  The location & formatting for the object to be written.
715  obj : object instance
716  The object to be written.
717  """
718  additionalData = butlerLocation.getAdditionalData()
719  locations = butlerLocation.getLocations()
720  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
721  logLoc = LogicalLocation(locationString, additionalData)
722  with open(logLoc.locString(), "wb") as outfile:
723  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
724 
725 
726 def readFitsCatalogStorage(butlerLocation):
727  """Read from a butlerLocation.
728 
729  Parameters
730  ----------
731  butlerLocation : ButlerLocation
732  The location & formatting for the object(s) to be read.
733 
734  Returns
735  -------
736  A list of objects as described by the butler location. One item for
737  each location in butlerLocation.getLocations()
738  """
739  pythonType = butlerLocation.getPythonType()
740  if pythonType is not None:
741  if isinstance(pythonType, basestring):
742  pythonType = doImport(pythonType)
743  results = []
744  additionalData = butlerLocation.getAdditionalData()
745  for locationString in butlerLocation.getLocations():
746  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
747  logLoc = LogicalLocation(locStringWithRoot, additionalData)
748  if not os.path.exists(logLoc.locString()):
749  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
750  kwds = {}
751  if additionalData.exists("hdu"):
752  kwds["hdu"] = additionalData.getInt("hdu")
753  if additionalData.exists("flags"):
754  kwds["flags"] = additionalData.getInt("flags")
755  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
756  results.append(finalItem)
757  return results
758 
759 
760 def writeFitsCatalogStorage(butlerLocation, obj):
761  """Writes an object to a location and persistence format specified by
762  ButlerLocation
763 
764  Parameters
765  ----------
766  butlerLocation : ButlerLocation
767  The location & formatting for the object to be written.
768  obj : object instance
769  The object to be written.
770  """
771  additionalData = butlerLocation.getAdditionalData()
772  locations = butlerLocation.getLocations()
773  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
774  logLoc = LogicalLocation(locationString, additionalData)
775  if additionalData.exists("flags"):
776  kwds = dict(flags=additionalData.getInt("flags"))
777  else:
778  kwds = {}
779  obj.writeFits(logLoc.locString(), **kwds)
780 
781 
782 def readMatplotlibStorage(butlerLocation):
783  """Read from a butlerLocation (always fails for this storage type).
784 
785  Parameters
786  ----------
787  butlerLocation : ButlerLocation
788  The location & formatting for the object(s) to be read.
789 
790  Returns
791  -------
792  A list of objects as described by the butler location. One item for
793  each location in butlerLocation.getLocations()
794  """
795  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
796 
797 
798 def writeMatplotlibStorage(butlerLocation, obj):
799  """Writes a matplotlib.figure.Figure to a location, using the template's
800  filename suffix to infer the file format.
801 
802  Parameters
803  ----------
804  butlerLocation : ButlerLocation
805  The location & formatting for the object to be written.
806  obj : matplotlib.figure.Figure
807  The object to be written.
808  """
809  additionalData = butlerLocation.getAdditionalData()
810  locations = butlerLocation.getLocations()
811  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
812  logLoc = LogicalLocation(locationString, additionalData)
813  # SafeFilename appends a random suffix, which corrupts the extension
814  # matplotlib uses to guess the file format.
815  # Instead, we extract the extension from the original location
816  # and pass that as the format directly.
817  _, ext = os.path.splitext(locations[0])
818  if ext:
819  ext = ext[1:] # strip off leading '.'
820  else:
821  # If there is no extension, we let matplotlib fall back to its
822  # default.
823  ext = None
824  obj.savefig(logLoc.locString(), format=ext)
825 
826 
827 def readPafStorage(butlerLocation):
828  """Read from a butlerLocation.
829 
830  Parameters
831  ----------
832  butlerLocation : ButlerLocation
833  The location & formatting for the object(s) to be read.
834 
835  Returns
836  -------
837  A list of objects as described by the butler location. One item for
838  each location in butlerLocation.getLocations()
839  """
840  results = []
841  for locationString in butlerLocation.getLocations():
842  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
843  butlerLocation.getAdditionalData())
844  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
845  results.append(finalItem)
846  return results
847 
848 
849 def readYamlStorage(butlerLocation):
850  """Read from a butlerLocation.
851 
852  Parameters
853  ----------
854  butlerLocation : ButlerLocation
855  The location & formatting for the object(s) to be read.
856 
857  Returns
858  -------
859  A list of objects as described by the butler location. One item for
860  each location in butlerLocation.getLocations()
861  """
862  results = []
863  for locationString in butlerLocation.getLocations():
864  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
865  butlerLocation.getAdditionalData())
866  finalItem = Policy(filePath=logLoc.locString())
867  results.append(finalItem)
868  return results
869 
870 
871 def readBoostStorage(butlerLocation):
872  results = []
873  additionalData = butlerLocation.getAdditionalData()
874  for locationString in butlerLocation.getLocations():
875  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
876  butlerLocation.getAdditionalData())
877  storageList = StorageList()
878  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
879  storageList.append(storage)
880  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
881  additionalData)
882  results.append(finalItem)
883  return results
884 
885 
886 def writeBoostStorage(butlerLocation, obj):
887  additionalData = butlerLocation.getAdditionalData()
888  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
889  with SafeFilename(location) as locationString:
890  logLoc = LogicalLocation(locationString, additionalData)
891  # Create a list of Storages for the item.
892  storageList = StorageList()
893  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
894  storageList.append(storage)
895  # Persist the item.
896  if hasattr(obj, '__deref__'):
897  # We have a smart pointer, so dereference it.
898  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
899  else:
900  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
901 
902 
903 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
904 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
905 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
906 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
907 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
908 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
909 PosixStorage.registerFormatters("PafStorage", writeFormatter=readPafStorage)
910 PosixStorage.registerFormatters("YamlStorage", readFormatter=readYamlStorage)
911 PosixStorage.registerFormatters("BoostStorage", readFitsStorage, writeFitsStorage)
912 
913 Storage.registerStorageClass(scheme='', cls=PosixStorage)
914 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)