Coverage for tests/test_s3.py: 24%
79 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-25 09:29 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-25 09:29 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import unittest
14from lsst.resources import ResourcePath
15from lsst.resources.s3utils import clean_test_environment
16from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase
18try:
19 import boto3
20 import botocore
21 from moto import mock_s3
22except ImportError:
23 boto3 = None
25 def mock_s3(cls):
26 """No-op decorator in case moto mock_s3 can not be imported."""
27 return cls
30class GenericS3TestCase(GenericTestCase, unittest.TestCase):
31 """Generic tests of S3 URIs."""
33 scheme = "s3"
34 netloc = "my_bucket"
37@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
38class S3ReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase):
39 """Tests of reading and writing S3 URIs."""
41 scheme = "s3"
42 netloc = "my_2nd_bucket"
44 mock_s3 = mock_s3()
45 """The mocked s3 interface from moto."""
47 def setUp(self):
48 # Enable S3 mocking of tests.
49 self.mock_s3.start()
51 clean_test_environment(self)
53 # set up some fake credentials if they do not exist
54 # self.usingDummyCredentials = setAwsEnvCredentials()
56 # MOTO needs to know that we expect Bucket bucketname to exist
57 s3 = boto3.resource("s3")
58 s3.create_bucket(Bucket=self.netloc)
60 super().setUp()
62 def tearDown(self):
63 s3 = boto3.resource("s3")
64 bucket = s3.Bucket(self.netloc)
65 try:
66 bucket.objects.all().delete()
67 except botocore.exceptions.ClientError as e:
68 if e.response["Error"]["Code"] == "404":
69 # the key was not reachable - pass
70 pass
71 else:
72 raise
74 bucket = s3.Bucket(self.netloc)
75 bucket.delete()
77 # Stop the S3 mock.
78 self.mock_s3.stop()
80 super().tearDown()
82 def test_bucket_fail(self):
83 # Deliberately create URI with unknown bucket.
84 uri = ResourcePath("s3://badbucket/something/")
86 with self.assertRaises(ValueError):
87 uri.mkdir()
89 with self.assertRaises(FileNotFoundError):
90 uri.remove()
92 def test_transfer_progress(self):
93 """Test progress bar reporting for upload and download."""
94 remote = self.root_uri.join("test.dat")
95 remote.write(b"42")
96 with ResourcePath.temporary_uri(suffix=".dat") as tmp:
97 # Download from S3.
98 with self.assertLogs("lsst.resources", level="DEBUG") as cm:
99 tmp.transfer_from(remote, transfer="auto")
100 self.assertRegex("".join(cm.output), r"test\.dat.*100\%")
102 # Upload to S3.
103 with self.assertLogs("lsst.resources", level="DEBUG") as cm:
104 remote.transfer_from(tmp, transfer="auto", overwrite=True)
105 self.assertRegex("".join(cm.output), rf"{tmp.basename()}.*100\%")
107 def test_handle(self):
108 remote = self.root_uri.join("test_handle.dat")
109 with remote.open("wb") as handle:
110 self.assertTrue(handle.writable())
111 # write 6 megabytes to make sure partial write work
112 handle.write(6 * 1024 * 1024 * b"a")
113 self.assertEqual(handle.tell(), 6 * 1024 * 1024)
114 handle.flush()
115 self.assertGreaterEqual(len(handle._multiPartUpload), 1)
117 # verify file can't be seeked back
118 with self.assertRaises(OSError):
119 handle.seek(0)
121 # write more bytes
122 handle.write(1024 * b"c")
124 # seek back and overwrite
125 handle.seek(6 * 1024 * 1024)
126 handle.write(1024 * b"b")
128 with remote.open("rb") as handle:
129 self.assertTrue(handle.readable())
130 # read the first 6 megabytes
131 result = handle.read(6 * 1024 * 1024)
132 self.assertEqual(result, 6 * 1024 * 1024 * b"a")
133 self.assertEqual(handle.tell(), 6 * 1024 * 1024)
134 # verify additional read gets the next part
135 result = handle.read(1024)
136 self.assertEqual(result, 1024 * b"b")
137 # see back to the beginning to verify seeking
138 handle.seek(0)
139 result = handle.read(1024)
140 self.assertEqual(result, 1024 * b"a")
143if __name__ == "__main__":
144 unittest.main()