lsst.daf.persistence  13.0-30-gd2bda26
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
storageInterface.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2017 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from abc import ABCMeta, abstractmethod
25 
26 
27 class NoRepositroyAtRoot(RuntimeError):
28  pass
29 
30 
32  """Defines the interface for a connection to a Storage location.
33 
34  Parameters
35  ----------
36  uri : string
37  URI or path that is used as the storage location.
38  create : bool
39  If True The StorageInterface subclass should create a new
40  repository at the root location. If False then a new repository
41  will not be created.
42 
43  Raises
44  ------
45  NoRepositroyAtRoot
46  If create is False and a repository does not exist at the root
47  specified by uri then NoRepositroyAtRoot is raised.
48  """
49  __metaclass__ = ABCMeta
50 
51  def __init__(self, uri, create):
52  """initialzer"""
53  pass
54 
55  formatters = {}
56 
57  @classmethod
58  def registerFormatter(cls, formatable, formatter):
59  """Register a formatter for a storageInterface subclass
60 
61  Parameters
62  ----------
63  cls : StorageInterface subclass
64  The type of StorageInterface the formatter is being registered for.
65  formatable : class object
66  The class object whose instances can be formatted by the formatter.
67  formatter : Formatter class
68  The formatter class that can be used by the StorageInterface instance to read and write the object
69  to the storage.
70 
71  Raises
72  ------
73  RuntimeError
74  For each object type and StorageInterface subclass a formatter should only be registered once. If
75  a second registration occurs a RuntimeError is raised.
76  """
77  classFormatters = StorageInterface.formatters.setdefault(cls, {})
78  if formatable in classFormatters:
79  raise RuntimeError(("Registration of second formatter {} for formattable class {} in " +
80  " storageInterface {}").format(formatter, formatable, cls))
81  classFormatters[formatable] = formatter
82 
83  @classmethod
84  def _getFormatter(cls, objType):
85  """Search in the registered formatters for the formatter for obj.
86 
87  Will attempt to find formatters registered for the objec type, and then
88  for base classes of the object in resolution order.
89 
90  Parameters
91  ----------
92  objType : class type
93  The type of class to find a formatter for.
94 
95  Returns
96  -------
97  formatter class object
98  The formatter class object to instantiate & use to read/write the object from/into the
99  storageInterface.
100  """
101  classFormatters = StorageInterface.formatters.get(cls, None)
102  if classFormatters is None:
103  return None
104  return classFormatters.get(objType, None)
105 
106  @abstractmethod
107  def write(self, butlerLocation, obj):
108  """Writes an object to a location and persistence format specified by ButlerLocation
109 
110  Parameters
111  ----------
112  butlerLocation : ButlerLocation
113  The location & formatting for the object to be written.
114  obj : object instance
115  The object to be written.
116  """
117 
118  @abstractmethod
119  def read(self, butlerLocation):
120  """Read from a butlerLocation.
121 
122  Parameters
123  ----------
124  butlerLocation : ButlerLocation
125  The location & formatting for the object(s) to be read.
126 
127  Returns
128  -------
129  A list of objects as described by the butler location. One item for
130  each location in butlerLocation.getLocations()
131  """
132 
133  @abstractmethod
134  def getLocalFile(self, path):
135  """Get a handle to a local copy of the file, downloading it to a
136  temporary if needed.
137 
138  Parameters
139  ----------
140  path : string
141  A path to the the file in storage, relative to root.
142 
143  Returns
144  -------
145  A handle to a local copy of the file. If storage is remote it will be
146  a temporary file. If storage is local it may be the original file or
147  a temporary file. The file name can be gotten via the 'name' property
148  of the returned object.
149  """
150 
151  @abstractmethod
152  def exists(self, location):
153  """Check if location exists.
154 
155  Parameters
156  ----------
157  location : ButlerLocation or string
158  A a string or a ButlerLocation that describes the location of an
159  object in this storage.
160 
161  Returns
162  -------
163  bool
164  True if exists, else False.
165  """
166 
167  @abstractmethod
168  def instanceSearch(self, path):
169  """Search for the given path in this storage instance.
170 
171  If the path contains an HDU indicator (a number in brackets before the
172  dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
173  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
174  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
175 
176  Parameters
177  ----------
178  path : string
179  A filename (and optionally prefix path) to search for within root.
180 
181  Returns
182  -------
183  string or None
184  The location that was found, or None if no location was found.
185  """
186 
187  @classmethod
188  @abstractmethod
189  def search(cls, root, path):
190  """Look for the given path in the current root.
191 
192  Also supports searching for the path in Butler v1 repositories by
193  following the Butler v1 _parent symlink
194 
195  If the path contains an HDU indicator (a number in brackets, e.g.
196  'foo.fits[1]', this will be stripped when searching and so
197  will match filenames without the HDU indicator, e.g. 'foo.fits'. The
198  path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
199 
200  Parameters
201  ----------
202  root : string
203  The path to the root directory.
204  path : string
205  The path to the file within the root directory.
206 
207  Returns
208  -------
209  string or None
210  The location that was found, or None if no location was found.
211  """
212 
213  @abstractmethod
214  def copyFile(self, fromLocation, toLocation):
215  """Copy a file from one location to another on the local filesystem.
216 
217  Parameters
218  ----------
219  fromLocation : string
220  Path and name of existing file.
221  toLocation : string
222  Path and name of new file.
223 
224  Returns
225  -------
226  None
227  """
228 
229  @abstractmethod
230  def locationWithRoot(self, location):
231  """Get the full path to the location.
232 
233  Parameters
234  ----------
235  location : string
236  Path to a location within the repository relative to repository
237  root.
238 
239  Returns
240  -------
241  string
242  Absolute path to to the locaiton within the repository.
243  """
244 
245  @classmethod
246  @abstractmethod
247  def getRepositoryCfg(cls, uri):
248  """Get a persisted RepositoryCfg
249 
250  Parameters
251  ----------
252  uri : URI or path to a RepositoryCfg
253  Description
254 
255  Returns
256  -------
257  A RepositoryCfg instance or None
258  """
259 
260  @classmethod
261  @abstractmethod
262  def putRepositoryCfg(cls, cfg, loc=None):
263  """Serialize a RepositoryCfg to a location.
264 
265  When loc == cfg.root, the RepositoryCfg is to be written at the root
266  location of the repository. In that case, root is not written, it is
267  implicit in the location of the cfg. This allows the cfg to move from
268  machine to machine without modification.
269 
270  Parameters
271  ----------
272  cfg : RepositoryCfg instance
273  The RepositoryCfg to be serailized.
274  loc : string, optional
275  The URI location (can be relative path) to write the RepositoryCfg.
276  If loc is None, the location will be read from the root parameter
277  of loc.
278 
279  Returns
280  -------
281  None
282  """
283 
284  @classmethod
285  @abstractmethod
286  def getMapperClass(cls, root):
287  """Get the mapper class associated with a repository root.
288 
289  Parameters
290  ----------
291  root : string
292  The location of a persisted RepositoryCfg is (new style repos).
293 
294  Returns
295  -------
296  A class object or a class instance, depending on the state of the
297  mapper when the repository was created.
298  """
299 
300  # Optional: Only needs to work if relative paths are sensical on this
301  # storage type and for the case where fromPath and toPath are of the same
302  # storage type.
303  @classmethod
304  def relativePath(cls, fromPath, toPath):
305  """Get a relative path from a location to a location.
306 
307  Parameters
308  ----------
309  fromPath : string
310  A path at which to start. It can be a relative path or an
311  absolute path.
312  toPath : string
313  A target location. It can be a relative path or an absolute path.
314 
315  Returns
316  -------
317  string
318  A relative path that describes the path from fromPath to toPath.
319  """
320  return toPath
321 
322  # Optional: Only needs to work if relative paths and absolute paths are
323  # sensical on this storage type and for the case where fromPath and toPath
324  # are of the same storage type.
325  @classmethod
326  def absolutePath(cls, fromPath, relativePath):
327  """Get an absolute path for the path from fromUri to toUri
328 
329  Parameters
330  ----------
331  fromPath : the starting location
332  A location at which to start. It can be a relative path or an
333  absolute path.
334  relativePath : the location relative to fromPath
335  A relative path.
336 
337  Returns
338  -------
339  string
340  Path that is an absolute path representation of fromPath +
341  relativePath, if one exists. If relativePath is absolute or if
342  fromPath is not related to relativePath then relativePath will be
343  returned.
344  """
345  return relativePath