lsst.daf.persistence  16.0-4-g32d12de
posixStorage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from past.builtins import basestring
25 import sys
26 import pickle
27 import importlib
28 import os
29 import urllib.parse
30 import glob
31 import shutil
32 import yaml
33 
34 from . import (LogicalLocation, Persistence, Policy, StorageList,
35  StorageInterface, Storage, ButlerLocation,
36  NoRepositroyAtRoot, RepositoryCfg, doImport)
37 from lsst.log import Log
38 import lsst.pex.policy as pexPolicy
39 from .safeFileIo import SafeFilename, safeMakeDir
40 
41 
42 __all__ = ["PosixStorage"]
43 
44 
46  """Defines the interface for a storage location on the local filesystem.
47 
48  Parameters
49  ----------
50  uri : string
51  URI or path that is used as the storage location.
52  create : bool
53  If True a new repository will be created at the root location if it
54  does not exist. If False then a new repository will not be created.
55 
56  Raises
57  ------
58  NoRepositroyAtRoot
59  If create is False and a repository does not exist at the root
60  specified by uri then NoRepositroyAtRoot is raised.
61  """
62 
63  def __init__(self, uri, create):
64  self.log = Log.getLogger("daf.persistence.butler")
65  self.root = self._pathFromURI(uri)
66  if self.root and not os.path.exists(self.root):
67  if not create:
68  raise NoRepositroyAtRoot("No repository at {}".format(uri))
69  safeMakeDir(self.root)
70 
72 
73  @staticmethod
75  # Always use an empty Persistence policy until we can get rid of it
76  persistencePolicy = pexPolicy.Policy()
77  return Persistence.getPersistence(persistencePolicy)
78 
79  def __repr__(self):
80  return 'PosixStorage(root=%s)' % self.root
81 
82  @staticmethod
83  def _pathFromURI(uri):
84  """Get the path part of the URI"""
85  return urllib.parse.urlparse(uri).path
86 
87  @staticmethod
88  def relativePath(fromPath, toPath):
89  """Get a relative path from a location to a location.
90 
91  Parameters
92  ----------
93  fromPath : string
94  A path at which to start. It can be a relative path or an
95  absolute path.
96  toPath : string
97  A target location. It can be a relative path or an absolute path.
98 
99  Returns
100  -------
101  string
102  A relative path that describes the path from fromPath to toPath.
103  """
104  fromPath = os.path.realpath(fromPath)
105  return os.path.relpath(toPath, fromPath)
106 
107  @staticmethod
108  def absolutePath(fromPath, relativePath):
109  """Get an absolute path for the path from fromUri to toUri
110 
111  Parameters
112  ----------
113  fromPath : the starting location
114  A location at which to start. It can be a relative path or an
115  absolute path.
116  relativePath : the location relative to fromPath
117  A relative path.
118 
119  Returns
120  -------
121  string
122  Path that is an absolute path representation of fromPath +
123  relativePath, if one exists. If relativePath is absolute or if
124  fromPath is not related to relativePath then relativePath will be
125  returned.
126  """
127  if os.path.isabs(relativePath):
128  return relativePath
129  fromPath = os.path.realpath(fromPath)
130  return os.path.normpath(os.path.join(fromPath, relativePath))
131 
132  @staticmethod
134  """Get a persisted RepositoryCfg
135 
136  Parameters
137  ----------
138  uri : URI or path to a RepositoryCfg
139  Description
140 
141  Returns
142  -------
143  A RepositoryCfg instance or None
144  """
145  storage = Storage.makeFromURI(uri)
146  location = ButlerLocation(pythonType=RepositoryCfg,
147  cppType=None,
148  storageName=None,
149  locationList='repositoryCfg.yaml',
150  dataId={},
151  mapper=None,
152  storage=storage,
153  usedDataId=None,
154  datasetType=None)
155  return storage.read(location)
156 
157  @staticmethod
158  def putRepositoryCfg(cfg, loc=None):
159  storage = Storage.makeFromURI(cfg.root if loc is None else loc, create=True)
160  location = ButlerLocation(pythonType=RepositoryCfg,
161  cppType=None,
162  storageName=None,
163  locationList='repositoryCfg.yaml',
164  dataId={},
165  mapper=None,
166  storage=storage,
167  usedDataId=None,
168  datasetType=None)
169  storage.write(location, cfg)
170 
171  @staticmethod
172  def getMapperClass(root):
173  """Get the mapper class associated with a repository root.
174 
175  Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
176  new code and repositories; they should use the Repository parentCfg mechanism.
177 
178  Parameters
179  ----------
180  root : string
181  The location of a persisted ReositoryCfg is (new style repos), or
182  the location where a _mapper file is (old style repos).
183 
184  Returns
185  -------
186  A class object or a class instance, depending on the state of the
187  mapper when the repository was created.
188  """
189  if not (root):
190  return None
191 
192  cfg = PosixStorage.getRepositoryCfg(root)
193  if cfg is not None:
194  return cfg.mapper
195 
196  # Find a "_mapper" file containing the mapper class name
197  basePath = root
198  mapperFile = "_mapper"
199  while not os.path.exists(os.path.join(basePath, mapperFile)):
200  # Break abstraction by following _parent links from CameraMapper
201  if os.path.exists(os.path.join(basePath, "_parent")):
202  basePath = os.path.join(basePath, "_parent")
203  else:
204  mapperFile = None
205  break
206 
207  if mapperFile is not None:
208  mapperFile = os.path.join(basePath, mapperFile)
209 
210  # Read the name of the mapper class and instantiate it
211  with open(mapperFile, "r") as f:
212  mapperName = f.readline().strip()
213  components = mapperName.split(".")
214  if len(components) <= 1:
215  raise RuntimeError("Unqualified mapper name %s in %s" %
216  (mapperName, mapperFile))
217  pkg = importlib.import_module(".".join(components[:-1]))
218  return getattr(pkg, components[-1])
219 
220  return None
221 
222  @staticmethod
224  """For Butler V1 Repositories only, if a _parent symlink exists, get the location pointed to by the
225  symlink.
226 
227  Parameters
228  ----------
229  root : string
230  A path to the folder on the local filesystem.
231 
232  Returns
233  -------
234  string or None
235  A path to the parent folder indicated by the _parent symlink, or None if there is no _parent
236  symlink at root.
237  """
238  linkpath = os.path.join(root, '_parent')
239  if os.path.exists(linkpath):
240  try:
241  return os.readlink(os.path.join(root, '_parent'))
242  except OSError:
243  # some of the unit tests rely on a folder called _parent instead of a symlink to aother
244  # location. Allow that; return the path of that folder.
245  return os.path.join(root, '_parent')
246  return None
247 
248  def write(self, butlerLocation, obj):
249  """Writes an object to a location and persistence format specified by
250  ButlerLocation
251 
252  Parameters
253  ----------
254  butlerLocation : ButlerLocation
255  The location & formatting for the object to be written.
256  obj : object instance
257  The object to be written.
258  """
259  self.log.debug("Put location=%s obj=%s", butlerLocation, obj)
260 
261  writeFormatter = self.getWriteFormatter(butlerLocation.getStorageName())
262  if not writeFormatter:
263  writeFormatter = self.getWriteFormatter(butlerLocation.getPythonType())
264  if writeFormatter:
265  writeFormatter(butlerLocation, obj)
266  return
267 
268  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
269 
270  def read(self, butlerLocation):
271  """Read from a butlerLocation.
272 
273  Parameters
274  ----------
275  butlerLocation : ButlerLocation
276  The location & formatting for the object(s) to be read.
277 
278  Returns
279  -------
280  A list of objects as described by the butler location. One item for
281  each location in butlerLocation.getLocations()
282  """
283  readFormatter = self.getReadFormatter(butlerLocation.getStorageName())
284  if not readFormatter:
285  readFormatter = self.getReadFormatter(butlerLocation.getPythonType())
286  if readFormatter:
287  return readFormatter(butlerLocation)
288 
289  raise(RuntimeError("No formatter for location:{}".format(butlerLocation)))
290 
291  def butlerLocationExists(self, location):
292  """Implementation of PosixStorage.exists for ButlerLocation objects.
293  """
294  storageName = location.getStorageName()
295  if storageName not in ('BoostStorage', 'FitsStorage', 'PafStorage',
296  'PickleStorage', 'ConfigStorage', 'FitsCatalogStorage',
297  'YamlStorage', 'ParquetStorage', 'MatplotlibStorage'):
298  self.log.warn("butlerLocationExists for non-supported storage %s" % location)
299  return False
300  for locationString in location.getLocations():
301  logLoc = LogicalLocation(locationString, location.getAdditionalData()).locString()
302  obj = self.instanceSearch(path=logLoc)
303  if obj:
304  return True
305  return False
306 
307  def exists(self, location):
308  """Check if location exists.
309 
310  Parameters
311  ----------
312  location : ButlerLocation or string
313  A a string or a ButlerLocation that describes the location of an
314  object in this storage.
315 
316  Returns
317  -------
318  bool
319  True if exists, else False.
320  """
321  if isinstance(location, ButlerLocation):
322  return self.butlerLocationExists(location)
323 
324  obj = self.instanceSearch(path=location)
325  return bool(obj)
326 
327  def locationWithRoot(self, location):
328  """Get the full path to the location.
329 
330  :param location:
331  :return:
332  """
333  return os.path.join(self.root, location)
334 
335  @staticmethod
336  def v1RepoExists(root):
337  """Test if a Version 1 Repository exists.
338 
339  Version 1 Repositories only exist in posix storages, do not have a
340  RepositoryCfg file, and contain either a registry.sqlite3 file, a
341  _mapper file, or a _parent link.
342 
343  Parameters
344  ----------
345  root : string
346  A path to a folder on the local filesystem.
347 
348  Returns
349  -------
350  bool
351  True if the repository at root exists, else False.
352  """
353  return os.path.exists(root) and (
354  os.path.exists(os.path.join(root, "registry.sqlite3")) or
355  os.path.exists(os.path.join(root, "_mapper")) or
356  os.path.exists(os.path.join(root, "_parent"))
357  )
358 
359  def copyFile(self, fromLocation, toLocation):
360  """Copy a file from one location to another on the local filesystem.
361 
362  Parameters
363  ----------
364  fromLocation : path
365  Path and name of existing file.
366  toLocation : path
367  Path and name of new file.
368 
369  Returns
370  -------
371  None
372  """
373  shutil.copy(os.path.join(self.root, fromLocation), os.path.join(self.root, toLocation))
374 
375  def getLocalFile(self, path):
376  """Get a handle to a local copy of the file, downloading it to a
377  temporary if needed.
378 
379  Parameters
380  ----------
381  A path the the file in storage, relative to root.
382 
383  Returns
384  -------
385  A handle to a local copy of the file. If storage is remote it will be
386  a temporary file. If storage is local it may be the original file or
387  a temporary file. The file name can be gotten via the 'name' property
388  of the returned object.
389  """
390  p = os.path.join(self.root, path)
391  try:
392  return open(p)
393  except IOError as e:
394  if e.errno == 2: # 'No such file or directory'
395  return None
396  else:
397  raise e
398 
399  def instanceSearch(self, path):
400  """Search for the given path in this storage instance.
401 
402  If the path contains an HDU indicator (a number in brackets before the
403  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
404  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
405  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
406 
407  Parameters
408  ----------
409  path : string
410  A filename (and optionally prefix path) to search for within root.
411 
412  Returns
413  -------
414  string or None
415  The location that was found, or None if no location was found.
416  """
417  return self.search(self.root, path)
418 
419  @staticmethod
420  def search(root, path, searchParents=False):
421  """Look for the given path in the current root.
422 
423  Also supports searching for the path in Butler v1 repositories by
424  following the Butler v1 _parent symlink
425 
426  If the path contains an HDU indicator (a number in brackets, e.g.
427  'foo.fits[1]', this will be stripped when searching and so
428  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
429  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
430 
431  Parameters
432  ----------
433  root : string
434  The path to the root directory.
435  path : string
436  The path to the file within the root directory.
437  searchParents : bool, optional
438  For Butler v1 repositories only, if true and a _parent symlink
439  exists, then the directory at _parent will be searched if the file
440  is not found in the root repository. Will continue searching the
441  parent of the parent until the file is found or no additional
442  parent exists.
443 
444  Returns
445  -------
446  string or None
447  The location that was found, or None if no location was found.
448  """
449  # Separate path into a root-equivalent prefix (in dir) and the rest
450  # (left in path)
451  rootDir = root
452  # First remove trailing slashes (#2527)
453  while len(rootDir) > 1 and rootDir[-1] == '/':
454  rootDir = rootDir[:-1]
455 
456  if path.startswith(rootDir + "/"):
457  # Common case; we have the same root prefix string
458  path = path[len(rootDir + '/'):]
459  pathPrefix = rootDir
460  elif rootDir == "/" and path.startswith("/"):
461  path = path[1:]
462  pathPrefix = None
463  else:
464  # Search for prefix that is the same as root
465  pathPrefix = os.path.dirname(path)
466  while pathPrefix != "" and pathPrefix != "/":
467  if os.path.realpath(pathPrefix) == os.path.realpath(root):
468  break
469  pathPrefix = os.path.dirname(pathPrefix)
470  if pathPrefix == "/":
471  path = path[1:]
472  elif pathPrefix != "":
473  path = path[len(pathPrefix)+1:]
474 
475  # Now search for the path in the root or its parents
476  # Strip off any cfitsio bracketed extension if present
477  strippedPath = path
478  pathStripped = None
479  firstBracket = path.find("[")
480  if firstBracket != -1:
481  strippedPath = path[:firstBracket]
482  pathStripped = path[firstBracket:]
483 
484  dir = rootDir
485  while True:
486  paths = glob.glob(os.path.join(dir, strippedPath))
487  if len(paths) > 0:
488  if pathPrefix != rootDir:
489  paths = [p[len(rootDir+'/'):] for p in paths]
490  if pathStripped is not None:
491  paths = [p + pathStripped for p in paths]
492  return paths
493  if searchParents:
494  dir = os.path.join(dir, "_parent")
495  if not os.path.exists(dir):
496  return None
497  else:
498  return None
499 
500  @staticmethod
501  def storageExists(uri):
502  """Ask if a storage at the location described by uri exists
503 
504  Parameters
505  ----------
506  root : string
507  URI to the the root location of the storage
508 
509  Returns
510  -------
511  bool
512  True if the storage exists, false if not
513  """
514  return os.path.exists(PosixStorage._pathFromURI(uri))
515 
516 
517 def readConfigStorage(butlerLocation):
518  """Read an lsst.pex.config.Config from a butlerLocation.
519 
520  Parameters
521  ----------
522  butlerLocation : ButlerLocation
523  The location for the object(s) to be read.
524 
525  Returns
526  -------
527  A list of objects as described by the butler location. One item for
528  each location in butlerLocation.getLocations()
529  """
530  results = []
531  for locationString in butlerLocation.getLocations():
532  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
533  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
534  if not os.path.exists(logLoc.locString()):
535  raise RuntimeError("No such config file: " + logLoc.locString())
536  pythonType = butlerLocation.getPythonType()
537  if pythonType is not None:
538  if isinstance(pythonType, basestring):
539  pythonType = doImport(pythonType)
540  finalItem = pythonType()
541  finalItem.load(logLoc.locString())
542  results.append(finalItem)
543  return results
544 
545 
546 def writeConfigStorage(butlerLocation, obj):
547  """Writes an lsst.pex.config.Config object to a location specified by
548  ButlerLocation.
549 
550  Parameters
551  ----------
552  butlerLocation : ButlerLocation
553  The location for the object to be written.
554  obj : object instance
555  The object to be written.
556  """
557  filename = os.path.join(butlerLocation.getStorage().root, butlerLocation.getLocations()[0])
558  with SafeFilename(filename) as locationString:
559  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
560  obj.save(logLoc.locString())
561 
562 
563 def readFitsStorage(butlerLocation):
564  """Read a FITS image from a butlerLocation.
565 
566  Parameters
567  ----------
568  butlerLocation : ButlerLocation
569  The location for the object(s) to be read.
570 
571  Returns
572  -------
573  A list of objects as described by the butler location. One item for
574  each location in butlerLocation.getLocations()
575  """
576  results = []
577  for locationString in butlerLocation.getLocations():
578  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
579  logLoc = LogicalLocation(locStringWithRoot, butlerLocation.getAdditionalData())
580  storageList = StorageList()
581  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(),
582  logLoc)
583  storageList.append(storage)
584  finalItem = PosixStorage.getPersistence().unsafeRetrieve(
585  butlerLocation.getCppType(), storageList, butlerLocation.getAdditionalData())
586  results.append(finalItem)
587  return results
588 
589 
590 def writeFitsStorage(butlerLocation, obj):
591  """Writes an object to a FITS file specified by ButlerLocation.
592 
593  Parameters
594  ----------
595  butlerLocation : ButlerLocation
596  The location for the object to be written.
597  obj : object instance
598  The object to be written.
599  """
600  location = butlerLocation.getLocations()[0]
601  with SafeFilename(os.path.join(butlerLocation.getStorage().root, location)) as locationString:
602  logLoc = LogicalLocation(locationString, butlerLocation.getAdditionalData())
603  # Create a list of Storages for the item.
604  storageList = StorageList()
605  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
606  storageList.append(storage)
607  persistence = PosixStorage.getPersistence()
608  if hasattr(obj, '__deref__'):
609  # We have a smart pointer, so dereference it.
610  persistence.persist(obj.__deref__(), storageList, butlerLocation.getAdditionalData())
611  else:
612  persistence.persist(obj, storageList, butlerLocation.getAdditionalData())
613 
614 
615 def readParquetStorage(butlerLocation):
616  """Read a catalog from a Parquet file specified by ButlerLocation.
617 
618  The object returned by this is expected to be a subtype
619  of `ParquetTable`, which is a thin wrapper to `pyarrow.ParquetFile`
620  that allows for lazy loading of the data.
621 
622  Parameters
623  ----------
624  butlerLocation : ButlerLocation
625  The location for the object(s) to be read.
626 
627  Returns
628  -------
629  A list of objects as described by the butler location. One item for
630  each location in butlerLocation.getLocations()
631  """
632  results = []
633  additionalData = butlerLocation.getAdditionalData()
634 
635  for locationString in butlerLocation.getLocations():
636  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
637  logLoc = LogicalLocation(locStringWithRoot, additionalData)
638  if not os.path.exists(logLoc.locString()):
639  raise RuntimeError("No such parquet file: " + logLoc.locString())
640 
641  pythonType = butlerLocation.getPythonType()
642  if pythonType is not None:
643  if isinstance(pythonType, basestring):
644  pythonType = doImport(pythonType)
645 
646  filename = logLoc.locString()
647 
648  # pythonType will be ParquetTable (or perhaps MultilevelParquetTable)
649  # filename should be the first kwarg, but being explicit here.
650  results.append(pythonType(filename=filename))
651 
652  return results
653 
654 
655 def writeParquetStorage(butlerLocation, obj):
656  """Writes pandas dataframe to parquet file.
657 
658  Parameters
659  ----------
660  butlerLocation : ButlerLocation
661  The location for the object(s) to be read.
662  obj : `lsst.qa.explorer.parquetTable.ParquetTable`
663  Wrapped DataFrame to write.
664 
665  """
666  additionalData = butlerLocation.getAdditionalData()
667  locations = butlerLocation.getLocations()
668  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
669  logLoc = LogicalLocation(locationString, additionalData)
670  filename = logLoc.locString()
671  obj.write(filename)
672 
673 
674 def writeYamlStorage(butlerLocation, obj):
675  """Writes an object to a YAML file specified by ButlerLocation.
676 
677  Parameters
678  ----------
679  butlerLocation : ButlerLocation
680  The location for the object to be written.
681  obj : object instance
682  The object to be written.
683  """
684  additionalData = butlerLocation.getAdditionalData()
685  locations = butlerLocation.getLocations()
686  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
687  logLoc = LogicalLocation(locationString, additionalData)
688  with open(logLoc.locString(), "w") as outfile:
689  yaml.dump(obj, outfile)
690 
691 
692 def readPickleStorage(butlerLocation):
693  """Read an object from a pickle file specified by ButlerLocation.
694 
695  Parameters
696  ----------
697  butlerLocation : ButlerLocation
698  The location for the object(s) to be read.
699 
700  Returns
701  -------
702  A list of objects as described by the butler location. One item for
703  each location in butlerLocation.getLocations()
704  """
705  # Create a list of Storages for the item.
706  results = []
707  additionalData = butlerLocation.getAdditionalData()
708  for locationString in butlerLocation.getLocations():
709  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
710  logLoc = LogicalLocation(locStringWithRoot, additionalData)
711  if not os.path.exists(logLoc.locString()):
712  raise RuntimeError("No such pickle file: " + logLoc.locString())
713  with open(logLoc.locString(), "rb") as infile:
714  # py3: We have to specify encoding since some files were written
715  # by python2, and 'latin1' manages that conversion safely. See:
716  # http://stackoverflow.com/questions/28218466/unpickling-a-python-2-object-with-python-3/28218598#28218598
717  if sys.version_info.major >= 3:
718  finalItem = pickle.load(infile, encoding="latin1")
719  else:
720  finalItem = pickle.load(infile)
721  results.append(finalItem)
722  return results
723 
724 
725 def writePickleStorage(butlerLocation, obj):
726  """Writes an object to a pickle file specified by ButlerLocation.
727 
728  Parameters
729  ----------
730  butlerLocation : ButlerLocation
731  The location for the object to be written.
732  obj : object instance
733  The object to be written.
734  """
735  additionalData = butlerLocation.getAdditionalData()
736  locations = butlerLocation.getLocations()
737  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
738  logLoc = LogicalLocation(locationString, additionalData)
739  with open(logLoc.locString(), "wb") as outfile:
740  pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)
741 
742 
743 def readFitsCatalogStorage(butlerLocation):
744  """Read a catalog from a FITS table specified by ButlerLocation.
745 
746  Parameters
747  ----------
748  butlerLocation : ButlerLocation
749  The location for the object(s) to be read.
750 
751  Returns
752  -------
753  A list of objects as described by the butler location. One item for
754  each location in butlerLocation.getLocations()
755  """
756  pythonType = butlerLocation.getPythonType()
757  if pythonType is not None:
758  if isinstance(pythonType, basestring):
759  pythonType = doImport(pythonType)
760  results = []
761  additionalData = butlerLocation.getAdditionalData()
762  for locationString in butlerLocation.getLocations():
763  locStringWithRoot = os.path.join(butlerLocation.getStorage().root, locationString)
764  logLoc = LogicalLocation(locStringWithRoot, additionalData)
765  if not os.path.exists(logLoc.locString()):
766  raise RuntimeError("No such FITS catalog file: " + logLoc.locString())
767  kwds = {}
768  if additionalData.exists("hdu"):
769  kwds["hdu"] = additionalData.getInt("hdu")
770  if additionalData.exists("flags"):
771  kwds["flags"] = additionalData.getInt("flags")
772  finalItem = pythonType.readFits(logLoc.locString(), **kwds)
773  results.append(finalItem)
774  return results
775 
776 
777 def writeFitsCatalogStorage(butlerLocation, obj):
778  """Writes a catalog to a FITS table specified by ButlerLocation.
779 
780  Parameters
781  ----------
782  butlerLocation : ButlerLocation
783  The location for the object to be written.
784  obj : object instance
785  The object to be written.
786  """
787  additionalData = butlerLocation.getAdditionalData()
788  locations = butlerLocation.getLocations()
789  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
790  logLoc = LogicalLocation(locationString, additionalData)
791  if additionalData.exists("flags"):
792  kwds = dict(flags=additionalData.getInt("flags"))
793  else:
794  kwds = {}
795  obj.writeFits(logLoc.locString(), **kwds)
796 
797 
798 def readMatplotlibStorage(butlerLocation):
799  """Read from a butlerLocation (always fails for this storage type).
800 
801  Parameters
802  ----------
803  butlerLocation : ButlerLocation
804  The location for the object(s) to be read.
805 
806  Returns
807  -------
808  A list of objects as described by the butler location. One item for
809  each location in butlerLocation.getLocations()
810  """
811  raise NotImplementedError("Figures saved with MatplotlibStorage cannot be retreived using the Butler.")
812 
813 
814 def writeMatplotlibStorage(butlerLocation, obj):
815  """Writes a matplotlib.figure.Figure to a location, using the template's
816  filename suffix to infer the file format.
817 
818  Parameters
819  ----------
820  butlerLocation : ButlerLocation
821  The location for the object to be written.
822  obj : matplotlib.figure.Figure
823  The object to be written.
824  """
825  additionalData = butlerLocation.getAdditionalData()
826  locations = butlerLocation.getLocations()
827  with SafeFilename(os.path.join(butlerLocation.getStorage().root, locations[0])) as locationString:
828  logLoc = LogicalLocation(locationString, additionalData)
829  # SafeFilename appends a random suffix, which corrupts the extension
830  # matplotlib uses to guess the file format.
831  # Instead, we extract the extension from the original location
832  # and pass that as the format directly.
833  _, ext = os.path.splitext(locations[0])
834  if ext:
835  ext = ext[1:] # strip off leading '.'
836  else:
837  # If there is no extension, we let matplotlib fall back to its
838  # default.
839  ext = None
840  obj.savefig(logLoc.locString(), format=ext)
841 
842 
843 def readPafStorage(butlerLocation):
844  """Read a policy from a PAF file specified by a ButlerLocation.
845 
846  Parameters
847  ----------
848  butlerLocation : ButlerLocation
849  The location for the object(s) to be read.
850 
851  Returns
852  -------
853  A list of objects as described by the butler location. One item for
854  each location in butlerLocation.getLocations()
855  """
856  results = []
857  for locationString in butlerLocation.getLocations():
858  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
859  butlerLocation.getAdditionalData())
860  finalItem = pexPolicy.Policy.createPolicy(logLoc.locString())
861  results.append(finalItem)
862  return results
863 
864 
865 def readYamlStorage(butlerLocation):
866  """Read an object from a YAML file specified by a butlerLocation.
867 
868  Parameters
869  ----------
870  butlerLocation : ButlerLocation
871  The location for the object(s) to be read.
872 
873  Returns
874  -------
875  A list of objects as described by the butler location. One item for
876  each location in butlerLocation.getLocations()
877  """
878  results = []
879  for locationString in butlerLocation.getLocations():
880  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
881  butlerLocation.getAdditionalData())
882  if not os.path.exists(logLoc.locString()):
883  raise RuntimeError("No such YAML file: " + logLoc.locString())
884  # Butler Gen2 repository configurations are handled specially
885  if butlerLocation.pythonType == 'lsst.daf.persistence.RepositoryCfg':
886  finalItem = Policy(filePath=logLoc.locString())
887  else:
888  with open(logLoc.locString(), "rb") as infile:
889  finalItem = yaml.load(infile)
890  results.append(finalItem)
891  return results
892 
893 
894 def readBoostStorage(butlerLocation):
895  """Read an object from a boost::serialization file.
896 
897  Parameters
898  ----------
899  butlerLocation : ButlerLocation
900  The location for the object(s) to be read.
901 
902  Returns
903  -------
904  A list of objects as described by the butler location. One item for
905  each location in butlerLocation.getLocations()
906  """
907  results = []
908  additionalData = butlerLocation.getAdditionalData()
909  for locationString in butlerLocation.getLocations():
910  logLoc = LogicalLocation(butlerLocation.getStorage().locationWithRoot(locationString),
911  butlerLocation.getAdditionalData())
912  storageList = StorageList()
913  storage = PosixStorage.getPersistence().getRetrieveStorage(butlerLocation.getStorageName(), logLoc)
914  storageList.append(storage)
915  finalItem = PosixStorage.getPersistence().unsafeRetrieve(butlerLocation.getCppType(), storageList,
916  additionalData)
917  results.append(finalItem)
918  return results
919 
920 
921 def writeBoostStorage(butlerLocation, obj):
922  """Writes an object via boost::serialization.
923 
924  Parameters
925  ----------
926  butlerLocation : ButlerLocation
927  The location for the object to be written.
928  obj : object instance
929  The object to be written.
930  """
931  additionalData = butlerLocation.getAdditionalData()
932  location = butlerLocation.getStorage().locationWithRoot(butlerLocation.getLocations()[0])
933  with SafeFilename(location) as locationString:
934  logLoc = LogicalLocation(locationString, additionalData)
935  # Create a list of Storages for the item.
936  storageList = StorageList()
937  storage = PosixStorage.getPersistence().getPersistStorage(butlerLocation.getStorageName(), logLoc)
938  storageList.append(storage)
939  # Persist the item.
940  if hasattr(obj, '__deref__'):
941  # We have a smart pointer, so dereference it.
942  PosixStorage.getPersistence().persist(obj.__deref__(), storageList, additionalData)
943  else:
944  PosixStorage.getPersistence().persist(obj, storageList, additionalData)
945 
946 
947 PosixStorage.registerFormatters("FitsStorage", readFitsStorage, writeFitsStorage)
948 PosixStorage.registerFormatters("ParquetStorage", readParquetStorage, writeParquetStorage)
949 PosixStorage.registerFormatters("ConfigStorage", readConfigStorage, writeConfigStorage)
950 PosixStorage.registerFormatters("PickleStorage", readPickleStorage, writePickleStorage)
951 PosixStorage.registerFormatters("FitsCatalogStorage", readFitsCatalogStorage, writeFitsCatalogStorage)
952 PosixStorage.registerFormatters("MatplotlibStorage", readMatplotlibStorage, writeMatplotlibStorage)
953 PosixStorage.registerFormatters("PafStorage", readFormatter=readPafStorage)
954 PosixStorage.registerFormatters("YamlStorage", readYamlStorage, writeYamlStorage)
955 PosixStorage.registerFormatters("BoostStorage", readFitsStorage, writeFitsStorage)
956 
957 Storage.registerStorageClass(scheme='', cls=PosixStorage)
958 Storage.registerStorageClass(scheme='file', cls=PosixStorage)
def readMatplotlibStorage(butlerLocation)
def copyFile(self, fromLocation, toLocation)
def readConfigStorage(butlerLocation)
def readPickleStorage(butlerLocation)
def writeMatplotlibStorage(butlerLocation, obj)
Class for logical location of a persisted Persistable instance.
def writePickleStorage(butlerLocation, obj)
def readBoostStorage(butlerLocation)
def readParquetStorage(butlerLocation)
def writeFitsCatalogStorage(butlerLocation, obj)
def readPafStorage(butlerLocation)
def search(root, path, searchParents=False)
def writeParquetStorage(butlerLocation, obj)
def writeConfigStorage(butlerLocation, obj)
def readFitsStorage(butlerLocation)
def readFitsCatalogStorage(butlerLocation)
def doImport(pythonType)
Definition: utils.py:109
def readYamlStorage(butlerLocation)
def writeYamlStorage(butlerLocation, obj)
def writeBoostStorage(butlerLocation, obj)
def writeFitsStorage(butlerLocation, obj)