Coverage for tests/test_sqlite.py: 41%
152 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 12:04 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 12:04 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import os.path
24import stat
25import tempfile
26import unittest
27from contextlib import contextmanager
28from typing import Optional
30import sqlalchemy
31from lsst.daf.butler import ddl
32from lsst.daf.butler.registry import Registry
33from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
34from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
35from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
37TESTDIR = os.path.abspath(os.path.dirname(__file__))
40@contextmanager
41def removeWritePermission(filename):
42 mode = os.stat(filename).st_mode
43 try:
44 os.chmod(filename, stat.S_IREAD)
45 yield
46 finally:
47 os.chmod(filename, mode)
50def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
51 """Check whether we really can modify a database.
53 This intentionally allows any exception to be raised (not just
54 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
55 but the Database was initialized (incorrectly) with writeable=True.
56 """
57 try:
58 with database.declareStaticTables(create=True) as context:
59 table = context.addTable(
60 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
61 )
62 # Drop created table so that schema remains empty.
63 database._metadata.drop_all(database._engine, tables=[table])
64 return True
65 except Exception:
66 return False
69class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
70 """Tests for `SqliteDatabase` using a standard file-based database."""
72 def setUp(self):
73 self.root = makeTestTempDir(TESTDIR)
75 def tearDown(self):
76 removeTestTempDir(self.root)
78 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
79 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
80 engine = SqliteDatabase.makeEngine(filename=filename)
81 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
83 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
84 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable)
85 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable)
87 @contextmanager
88 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
89 with removeWritePermission(database.filename):
90 yield self.getNewConnection(database, writeable=False)
92 def testConnection(self):
93 """Test that different ways of connecting to a SQLite database
94 are equivalent.
95 """
96 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
97 # Create a read-write database by passing in the filename.
98 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0)
99 self.assertEqual(rwFromFilename.filename, filename)
100 self.assertEqual(rwFromFilename.origin, 0)
101 self.assertTrue(rwFromFilename.isWriteable())
102 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
103 # Create a read-write database via a URI.
104 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
105 self.assertEqual(rwFromUri.filename, filename)
106 self.assertEqual(rwFromUri.origin, 0)
107 self.assertTrue(rwFromUri.isWriteable())
108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
109 # We don't support SQLite URIs inside SQLAlchemy URIs.
110 with self.assertRaises(NotImplementedError):
111 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true")
113 # Test read-only connections against a read-only file.
114 with removeWritePermission(filename):
115 # Create a read-only database by passing in the filename.
116 roFromFilename = SqliteDatabase.fromEngine(
117 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False
118 )
119 self.assertEqual(roFromFilename.filename, filename)
120 self.assertEqual(roFromFilename.origin, 0)
121 self.assertFalse(roFromFilename.isWriteable())
122 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
123 # Create a read-write database via a URI.
124 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
125 self.assertEqual(roFromUri.filename, filename)
126 self.assertEqual(roFromUri.origin, 0)
127 self.assertFalse(roFromUri.isWriteable())
128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
130 def testTransactionLocking(self):
131 # This (inherited) test can't run on SQLite because of our use of an
132 # aggressive locking strategy there.
133 pass
136class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
137 """Tests for `SqliteDatabase` using an in-memory database."""
139 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
140 engine = SqliteDatabase.makeEngine(filename=None)
141 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
143 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
144 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable)
146 @contextmanager
147 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
148 yield self.getNewConnection(database, writeable=False)
150 def testConnection(self):
151 """Test that different ways of connecting to a SQLite database
152 are equivalent.
153 """
154 # Create an in-memory database by passing filename=None.
155 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0)
156 self.assertIsNone(memFromFilename.filename)
157 self.assertEqual(memFromFilename.origin, 0)
158 self.assertTrue(memFromFilename.isWriteable())
159 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
160 # Create an in-memory database via a URI.
161 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
162 self.assertIsNone(memFromUri.filename)
163 self.assertEqual(memFromUri.origin, 0)
164 self.assertTrue(memFromUri.isWriteable())
165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
166 # We don't support SQLite URIs inside SQLAlchemy URIs.
167 with self.assertRaises(NotImplementedError):
168 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true")
169 # We don't support read-only in-memory databases.
170 with self.assertRaises(NotImplementedError):
171 SqliteDatabase.makeEngine(filename=None, writeable=False)
173 def testTransactionLocking(self):
174 # This (inherited) test can't run on SQLite because of our use of an
175 # aggressive locking strategy there.
176 pass
179class SqliteFileRegistryTests(RegistryTests):
180 """Tests for `Registry` backed by a SQLite file-based database.
182 Note
183 ----
184 This is not a subclass of `unittest.TestCase` but to avoid repetition it
185 defines methods that override `unittest.TestCase` methods. To make this
186 work sublasses have to have this class first in the bases list.
187 """
189 def setUp(self):
190 self.root = makeTestTempDir(TESTDIR)
192 def tearDown(self):
193 removeTestTempDir(self.root)
195 @classmethod
196 def getDataDir(cls) -> str:
197 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
199 def makeRegistry(self, share_repo_with: Optional[Registry] = None) -> Registry:
200 if share_repo_with is None:
201 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
202 else:
203 filename = share_repo_with._db.filename
204 config = self.makeRegistryConfig()
205 config["db"] = f"sqlite:///{filename}"
206 if share_repo_with is None:
207 return Registry.createFromConfig(config, butlerRoot=self.root)
208 else:
209 return Registry.fromConfig(config, butlerRoot=self.root)
212class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase):
213 """Tests for `Registry` backed by a SQLite file-based database.
215 This test case uses NameKeyCollectionManager and
216 ByDimensionsDatasetRecordStorageManager.
217 """
219 def makeRegistry(self, share_repo_with: Optional[Registry] = None) -> Registry:
220 if share_repo_with is None:
221 with self.assertWarns(FutureWarning):
222 return super().makeRegistry()
223 else:
224 return super().makeRegistry(share_repo_with)
226 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
227 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager"
230class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
231 """Tests for `Registry` backed by a SQLite file-based database.
233 This test case uses NameKeyCollectionManager and
234 ByDimensionsDatasetRecordStorageManagerUUID.
235 """
237 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
238 datasetsManager = (
239 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
240 )
243class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
244 """Tests for `Registry` backed by a SQLite file-based database.
246 This test case uses SynthIntKeyCollectionManager and
247 ByDimensionsDatasetRecordStorageManagerUUID.
248 """
250 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
251 datasetsManager = (
252 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
253 )
256class SqliteMemoryRegistryTests(RegistryTests):
257 """Tests for `Registry` backed by a SQLite in-memory database."""
259 @classmethod
260 def getDataDir(cls) -> str:
261 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
263 def makeRegistry(self, share_repo_with: Optional[Registry] = None) -> Optional[Registry]:
264 if share_repo_with is not None:
265 return None
266 config = self.makeRegistryConfig()
267 config["db"] = "sqlite://"
268 return Registry.createFromConfig(config)
270 def testMissingAttributes(self):
271 """Test for instantiating a registry against outdated schema which
272 misses butler_attributes table.
273 """
274 # TODO: Once we have stable gen3 schema everywhere this test can be
275 # dropped (DM-27373).
276 config = self.makeRegistryConfig()
277 config["db"] = "sqlite://"
278 with self.assertRaises(sqlalchemy.exc.OperationalError):
279 Registry.fromConfig(config)
282class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
283 """Tests for `Registry` backed by a SQLite in-memory database.
285 This test case uses NameKeyCollectionManager and
286 ByDimensionsDatasetRecordStorageManagerUUID.
287 """
289 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
290 datasetsManager = (
291 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
292 )
295class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
296 """Tests for `Registry` backed by a SQLite in-memory database.
298 This test case uses SynthIntKeyCollectionManager and
299 ByDimensionsDatasetRecordStorageManagerUUID.
300 """
302 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
303 datasetsManager = (
304 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
305 )
308if __name__ == "__main__": 308 ↛ 309line 308 didn't jump to line 309, because the condition on line 308 was never true
309 unittest.main()