lsst.daf.persistence  13.0-30-gd2bda26+1
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
storage.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from __future__ import absolute_import
25 
26 from builtins import object
27 from future import standard_library
28 import urllib.parse
29 from . import NoRepositroyAtRoot
30 standard_library.install_aliases()
31 
32 
33 class Storage:
34  """Base class for storages"""
35 
36  storages = {}
37 
38  def __init__(self):
39  self.repositoryCfgs = {}
40 
41  @staticmethod
42  def registerStorageClass(scheme, cls):
43  """Register derived classes for lookup by URI scheme.
44 
45  A scheme is a name that describes the form a resource at the beginning of a URI
46  e.g. 'http' indicates HTML and related code, such as is found in http://www.lsst.org
47 
48  The only currently supported schemes are:
49  * 'file' where the portion of the URI after the // indicates an absolute locaiton on disk.
50  for example: file:/my_repository_folder/
51  * '' (no scheme) where the entire string is a relative path on the local system
52  for example "my_repository_folder" will indicate a folder in the current working directory with the
53  same name.
54 
55  See documentation for the urlparse python library for more information.
56 
57  .. warning::
58 
59  Storage is 'wet paint' and very likely to change during factorization of Butler back end and
60  storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is
61  strongly discouraged.
62 
63  Parameters
64  ----------
65  scheme : str
66  Name of the `scheme` the class is being registered for, which appears at the beginning of a URI.
67  cls : class object
68  A class object that should be used for a given scheme.
69  """
70  if scheme in Storage.storages:
71  raise RuntimeError("Scheme '%s' already registered:%s" % (scheme, Storage.storages[scheme]))
72  Storage.storages[scheme] = cls
73 
74  def getRepositoryCfg(self, uri):
75  """Get a RepositoryCfg from a location specified by uri.
76 
77  If a cfg is found then it is cached by the uri, so that multiple lookups
78  are not performed on storages that might be remote.
79 
80  RepositoryCfgs are not supposed to change once they are created so this
81  should not lead to stale data.
82  """
83  cfg = self.repositoryCfgs.get(uri, None)
84  if cfg:
85  return cfg
86  parseRes = urllib.parse.urlparse(uri)
87  if parseRes.scheme in Storage.storages:
88  cfg = Storage.storages[parseRes.scheme].getRepositoryCfg(uri)
89  if cfg:
90  self.repositoryCfgs[uri] = cfg
91  else:
92  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
93  return cfg
94 
95  @staticmethod
96  def putRepositoryCfg(cfg, uri):
97  """Write a RepositoryCfg object to a location described by uri"""
98  ret = None
99  parseRes = urllib.parse.urlparse(uri)
100  if parseRes.scheme in Storage.storages:
101  ret = Storage.storages[parseRes.scheme].putRepositoryCfg(cfg, uri)
102  else:
103  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
104  return ret
105 
106  @staticmethod
107  def getMapperClass(uri):
108  """Get a mapper class cfg value from location described by uri.
109 
110  Note that in legacy repositories the mapper may be specified by a file called _mapper at the uri
111  location, and in newer repositories the mapper would be specified by a RepositoryCfg stored at the uri
112  location.
113 
114  .. warning::
115 
116  Storage is 'wet paint' and very likely to change during factorization of Butler back end and
117  storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is
118  strongly discouraged.
119 
120  """
121  ret = None
122  parseRes = urllib.parse.urlparse(uri)
123  if parseRes.scheme in Storage.storages:
124  ret = Storage.storages[parseRes.scheme].getMapperClass(uri)
125  else:
126  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
127  return ret
128 
129  @staticmethod
130  def makeFromURI(uri, create=True):
131  '''Instantiate a StorageInterface sublcass from a URI.
132 
133  .. warning::
134 
135  Storage is 'wet paint' and very likely to change during factorization of Butler back end and
136  storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is
137  strongly discouraged.
138 
139  Parameters
140  ----------
141  uri : string
142  The uri to the root location of a repository.
143  create : bool, optional
144  If True The StorageInterface subclass should create a new
145  repository at the root location. If False then a new repository
146  will not be created.
147 
148  Returns
149  -------
150  A Storage subclass instance, or if create is False and a repository
151  does not exist at the root location then returns None.
152 
153  Raises
154  ------
155  RuntimeError
156  When a StorageInterface subclass does not exist for the scheme
157  indicated by the uri.
158  '''
159  ret = None
160  parseRes = urllib.parse.urlparse(uri)
161  if parseRes.scheme in Storage.storages:
162  theClass = Storage.storages[parseRes.scheme]
163  try:
164  ret = theClass(uri=uri, create=create)
165  except NoRepositroyAtRoot:
166  pass
167  else:
168  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
169  return ret
170 
171  @staticmethod
172  def isPosix(uri):
173  """Test if a URI is for a local filesystem storage.
174 
175  This is mostly for backward compatibility; Butler V1 repositories were only ever on the local
176  filesystem. They may exist but not have a RepositoryCfg class. This enables conditional checking for a
177  V1 Repository.
178 
179  This function treats 'file' and '' (no scheme) as posix storages, see
180  the class docstring for more details.
181 
182  Parameters
183  ----------
184  uri : string
185  URI to the root of a Repository.
186 
187  Returns
188  -------
189  Bool
190  True if the URI is associated with a posix storage, else false.
191  """
192  parseRes = urllib.parse.urlparse(uri)
193  if parseRes.scheme in ('file', ''):
194  return True
195  return False
196 
197  @staticmethod
198  def relativePath(fromUri, toUri):
199  """Get a relative path from a location to a location, if a relative path for these 2 locations exists.
200 
201  Parameters
202  ----------
203  fromPath : string
204  A URI that describes a location at which to start.
205  toPath : string
206  A URI that describes a target location.
207 
208  Returns
209  -------
210  string
211  A relative path that describes the path from fromUri to toUri, provided one exists. If a relative
212  path between the two URIs does not exist then the entire toUri path is returned.
213  """
214  fromUriParseRes = urllib.parse.urlparse(fromUri)
215  toUriParseRes = urllib.parse.urlparse(toUri)
216  if fromUriParseRes.scheme != toUriParseRes.scheme:
217  return toUri
218  storage = Storage.storages.get(fromUriParseRes.scheme, None)
219  if not storage:
220  return toUri
221  return storage.relativePath(fromUri, toUri)
222 
223  @staticmethod
224  def absolutePath(fromUri, toUri):
225  """Get an absolute path for the path from fromUri to toUri
226 
227  Parameters
228  ----------
229  fromUri : the starting location
230  Description
231  toUri : the location relative to fromUri
232  Description
233 
234  Returns
235  -------
236  string
237  URI that is absolutepath fromUri + toUri, if one exists. If toUri is absolute or if fromUri is not
238  related to toUri (e.g. are of different storage types) then toUri will be returned.
239  """
240  fromUriParseRes = urllib.parse.urlparse(fromUri)
241  toUriParseRes = urllib.parse.urlparse(toUri)
242  if fromUriParseRes.scheme != toUriParseRes.scheme:
243  return toUri
244  storage = Storage.storages.get(fromUriParseRes.scheme, None)
245  if not storage:
246  return toUri
247  return storage.absolutePath(fromUri, toUri)
248 
249  @staticmethod
250  def search(uri, path):
251  """Look for the given path in a storage root at URI; return None if it can't be found.
252 
253  If the path contains an HDU indicator (a number in brackets before the
254  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
255  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
256  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
257 
258 
259  Parameters
260  ----------
261  root : string
262  URI to the the root location to search
263  path : string
264  A filename (and optionally prefix path) to search for within root.
265 
266  Returns
267  -------
268  string or None
269  The location that was found, or None if no location was found.
270  """
271  parseRes = urllib.parse.urlparse(uri)
272  storage = Storage.storages.get(parseRes.scheme, None)
273  if storage:
274  return storage.search(uri, path)
275  return None
276 
277  @staticmethod
278  def storageExists(uri):
279  """Ask if a storage at the location described by uri exists
280 
281  Parameters
282  ----------
283  root : string
284  URI to the the root location of the storage
285 
286  Returns
287  -------
288  bool
289  True if the storage exists, false if not
290  """
291  parseRes = urllib.parse.urlparse(uri)
292  storage = Storage.storages.get(parseRes.scheme, None)
293  if storage:
294  return storage.storageExists(uri)
295  return None