Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""S3 datastore.""" 

25 

26__all__ = ("S3Datastore", ) 

27 

28import logging 

29 

30from typing import ( 

31 TYPE_CHECKING, 

32 Union, 

33) 

34 

35from .remoteFileDatastore import RemoteFileDatastore 

36from lsst.daf.butler.core._butlerUri.s3utils import getS3Client, bucketExists 

37 

38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true

39 from lsst.daf.butler import DatastoreConfig 

40 from lsst.daf.butler.registry.interfaces import DatastoreRegistryBridgeManager 

41 

42log = logging.getLogger(__name__) 

43 

44 

45class S3Datastore(RemoteFileDatastore): 

46 """Basic S3 Object Storage backed Datastore. 

47 

48 Parameters 

49 ---------- 

50 config : `DatastoreConfig` or `str` 

51 Configuration. A string should refer to the name of the config file. 

52 bridgeManager : `DatastoreRegistryBridgeManager` 

53 Object that manages the interface between `Registry` and datastores. 

54 butlerRoot : `str`, optional 

55 New datastore root to use to override the configuration value. 

56 

57 Raises 

58 ------ 

59 ValueError 

60 If root location does not exist and ``create`` is `False` in the 

61 configuration. 

62 

63 Notes 

64 ----- 

65 S3Datastore supports non-link transfer modes for file-based ingest: 

66 `"move"`, `"copy"`, and `None` (no transfer). 

67 """ 

68 

69 defaultConfigFile = "datastores/s3Datastore.yaml" 

70 """Path to configuration defaults. Accessed within the ``configs`` resource 

71 or relative to a search path. Can be None if no defaults specified. 

72 """ 

73 

74 def __init__(self, config: Union[DatastoreConfig, str], 

75 bridgeManager: DatastoreRegistryBridgeManager, butlerRoot: str = None): 

76 super().__init__(config, bridgeManager, butlerRoot) 

77 

78 self.client = getS3Client() 

79 if not bucketExists(self.locationFactory.netloc): 79 ↛ 85line 79 didn't jump to line 85, because the condition on line 79 was never true

80 # PosixDatastore creates the root directory if one does not exist. 

81 # Calling s3 client.create_bucket is possible but also requires 

82 # ACL LocationConstraints, Permissions and other configuration 

83 # parameters, so for now we do not create a bucket if one is 

84 # missing. Further discussion can make this happen though. 

85 raise IOError(f"Bucket {self.locationFactory.netloc} does not exist!")