lsst.daf.persistence  13.0-11-gfc17871
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
storageContinued.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from __future__ import absolute_import
25 
26 __all__ = []
27 
28 from future import standard_library
29 standard_library.install_aliases()
30 from builtins import object
31 
32 import urllib.parse
33 from .storage import Storage
34 
35 from lsst.utils import continueClass
36 
37 @continueClass
38 class Storage:
39  """Base class for storages"""
40 
41  storages = {}
42 
43  @staticmethod
44  def registerStorageClass(scheme, cls):
45  """Register derived classes for lookup by URI scheme.
46 
47  A scheme is a name that describes the form a resource at the beginning of a URI
48  e.g. 'http' indicates HTML and related code, such as is found in http://www.lsst.org
49 
50  The only currently supported schemes are:
51  * 'file' where the portion of the URI after the // indicates an absolute locaiton on disk.
52  for example: file:/my_repository_folder/
53  * '' (no scheme) where the entire string is a relative path on the local system
54  for example "my_repository_folder" will indicate a folder in the current working directory with the
55  same name.
56 
57  See documentation for the urlparse python library for more information.
58 
59  .. warning::
60 
61  Storage is 'wet paint' and very likely to change during factorization of Butler back end and
62  storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is
63  strongly discouraged.
64 
65  Parameters
66  ----------
67  scheme : str
68  Name of the `scheme` the class is being registered for, which appears at the beginning of a URI.
69  cls : class object
70  A class object that should be used for a given scheme.
71  """
72  if scheme in Storage.storages:
73  raise RuntimeError("Scheme '%s' already registered:%s" % (scheme, Storage.storages[scheme]))
74  Storage.storages[scheme] = cls
75 
76  @staticmethod
77  def getRepositoryCfg(uri):
78  """Get a RepositoryCfg from a location specified by uri."""
79  ret = None
80  parseRes = urllib.parse.urlparse(uri)
81  if parseRes.scheme in Storage.storages:
82  ret = Storage.storages[parseRes.scheme].getRepositoryCfg(uri)
83  else:
84  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
85  return ret
86 
87  @staticmethod
88  def putRepositoryCfg(cfg, uri):
89  """Write a RepositoryCfg object to a location described by uri"""
90  ret = None
91  parseRes = urllib.parse.urlparse(uri)
92  if parseRes.scheme in Storage.storages:
93  ret = Storage.storages[parseRes.scheme].putRepositoryCfg(cfg, uri)
94  else:
95  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
96  return ret
97 
98  @staticmethod
99  def getMapperClass(uri):
100  """Get a mapper class cfg value from location described by uri.
101 
102  Note that in legacy repositories the mapper may be specified by a file called _mapper at the uri
103  location, and in newer repositories the mapper would be specified by a RepositoryCfg stored at the uri
104  location.
105 
106  .. warning::
107 
108  Storage is 'wet paint' and very likely to change during factorization of Butler back end and
109  storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is
110  strongly discouraged.
111 
112  """
113  ret = None
114  parseRes = urllib.parse.urlparse(uri)
115  if parseRes.scheme in Storage.storages:
116  ret = Storage.storages[parseRes.scheme].getMapperClass(uri)
117  else:
118  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
119  return ret
120 
121  @staticmethod
122  def makeFromURI(uri):
123  '''Instantiate a storage sublcass from a URI.
124 
125  .. warning::
126 
127  Storage is 'wet paint' and very likely to change during factorization of Butler back end and
128  storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is
129  strongly discouraged.
130 
131  Parameters
132  ----------
133  uri : string
134  The uri to the root location of a repository.
135 
136  Returns
137  -------
138  A Storage subclass instance.
139  '''
140  ret = None
141  parseRes = urllib.parse.urlparse(uri)
142  if parseRes.scheme in Storage.storages:
143  theClass = Storage.storages[parseRes.scheme]
144  ret = theClass(uri=uri)
145  else:
146  raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme)
147  return ret
148 
149  @staticmethod
150  def isPosix(uri):
151  """Test if a URI is for a local filesystem storage.
152 
153  This is mostly for backward compatibility; Butler V1 repositories were only ever on the local
154  filesystem. They may exist but not have a RepositoryCfg class. This enables conditional checking for a
155  V1 Repository.
156 
157  This function treats 'file' and '' (no scheme) as posix storages, see
158  the class docstring for more details.
159 
160  Parameters
161  ----------
162  uri : string
163  URI to the root of a Repository.
164 
165  Returns
166  -------
167  Bool
168  True if the URI is associated with a posix storage, else false.
169  """
170  parseRes = urllib.parse.urlparse(uri)
171  if parseRes.scheme in ('file', ''):
172  return True
173  return False
174 
175  @staticmethod
176  def relativePath(fromUri, toUri):
177  """Get a relative path from a location to a location, if a relative path for these 2 locations exists.
178 
179  Parameters
180  ----------
181  fromPath : string
182  A URI that describes a location at which to start.
183  toPath : string
184  A URI that describes a target location.
185 
186  Returns
187  -------
188  string
189  A relative path that describes the path from fromUri to toUri, provided one exists. If a relative
190  path between the two URIs does not exist then the entire toUri path is returned.
191  """
192  fromUriParseRes = urllib.parse.urlparse(fromUri)
193  toUriParseRes = urllib.parse.urlparse(toUri)
194  if fromUriParseRes.scheme != toUriParseRes.scheme:
195  return toUri
196  storage = Storage.storages.get(fromUriParseRes.scheme, None)
197  if not storage:
198  return toUri
199  return storage.relativePath(fromUri, toUri)
200 
201  @staticmethod
202  def absolutePath(fromUri, toUri):
203  """Get an absolute path for the path from fromUri to toUri
204 
205  Parameters
206  ----------
207  fromUri : the starting location
208  Description
209  toUri : the location relative to fromUri
210  Description
211 
212  Returns
213  -------
214  string
215  URI that is absolutepath fromUri + toUri, if one exists. If toUri is absolute or if fromUri is not
216  related to toUri (e.g. are of different storage types) then toUri will be returned.
217  """
218  fromUriParseRes = urllib.parse.urlparse(fromUri)
219  toUriParseRes = urllib.parse.urlparse(toUri)
220  if fromUriParseRes.scheme != toUriParseRes.scheme:
221  return toUri
222  storage = Storage.storages.get(fromUriParseRes.scheme, None)
223  if not storage:
224  return toUri
225  return storage.absolutePath(fromUri, toUri)
226 
227  @staticmethod
228  def search(uri, path):
229  """Look for the given path in a storage root at URI; return None if it can't be found.
230 
231  If the path contains an HDU indicator (a number in brackets before the
232  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
233  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
234  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
235 
236 
237  Parameters
238  ----------
239  root : string
240  URI to the the root location to search
241  path : string
242  A filename (and optionally prefix path) to search for within root.
243 
244  Returns
245  -------
246  string or None
247  The location that was found, or None if no location was found.
248  """
249  parseRes = urllib.parse.urlparse(uri)
250  storage = Storage.storages.get(parseRes.scheme, None)
251  if storage:
252  return storage.search(uri, path)
253  return None
254 
255  @staticmethod
256  def storageExists(uri):
257  """Ask if a storage at the location described by uri exists
258 
259  Parameters
260  ----------
261  root : string
262  URI to the the root location of the storage
263 
264  Returns
265  -------
266  bool
267  True if the storage exists, false if not
268  """
269  parseRes = urllib.parse.urlparse(uri)
270  storage = Storage.storages.get(parseRes.scheme, None)
271  if storage:
272  return storage.storageExists(uri)
273  return None