Coverage for python/lsst/daf/butler/datastores/s3Datastore.py : 76%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24"""S3 datastore."""
26__all__ = ("S3Datastore", )
28import logging
30from typing import (
31 TYPE_CHECKING,
32 Union,
33)
35from .remoteFileDatastore import RemoteFileDatastore
36from lsst.daf.butler.core._butlerUri.s3utils import getS3Client, bucketExists
38if TYPE_CHECKING: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true
39 from lsst.daf.butler import DatastoreConfig
40 from lsst.daf.butler.registry.interfaces import DatastoreRegistryBridgeManager
42log = logging.getLogger(__name__)
45class S3Datastore(RemoteFileDatastore):
46 """Basic S3 Object Storage backed Datastore.
48 Parameters
49 ----------
50 config : `DatastoreConfig` or `str`
51 Configuration. A string should refer to the name of the config file.
52 bridgeManager : `DatastoreRegistryBridgeManager`
53 Object that manages the interface between `Registry` and datastores.
54 butlerRoot : `str`, optional
55 New datastore root to use to override the configuration value.
57 Raises
58 ------
59 ValueError
60 If root location does not exist and ``create`` is `False` in the
61 configuration.
63 Notes
64 -----
65 S3Datastore supports non-link transfer modes for file-based ingest:
66 `"move"`, `"copy"`, and `None` (no transfer).
67 """
69 defaultConfigFile = "datastores/s3Datastore.yaml"
70 """Path to configuration defaults. Accessed within the ``configs`` resource
71 or relative to a search path. Can be None if no defaults specified.
72 """
74 def __init__(self, config: Union[DatastoreConfig, str],
75 bridgeManager: DatastoreRegistryBridgeManager, butlerRoot: str = None):
76 super().__init__(config, bridgeManager, butlerRoot)
78 self.client = getS3Client()
79 if not bucketExists(self.locationFactory.netloc): 79 ↛ 85line 79 didn't jump to line 85, because the condition on line 79 was never true
80 # PosixDatastore creates the root directory if one does not exist.
81 # Calling s3 client.create_bucket is possible but also requires
82 # ACL LocationConstraints, Permissions and other configuration
83 # parameters, so for now we do not create a bucket if one is
84 # missing. Further discussion can make this happen though.
85 raise IOError(f"Bucket {self.locationFactory.netloc} does not exist!")