Coverage for tests/test_sqlite.py: 39%
144 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-21 09:55 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-21 09:55 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import os.path
24import stat
25import tempfile
26import unittest
27from contextlib import contextmanager
29import sqlalchemy
30from lsst.daf.butler import ddl
31from lsst.daf.butler.registry import _ButlerRegistry, _RegistryFactory
32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
33from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
34from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
36TESTDIR = os.path.abspath(os.path.dirname(__file__))
39@contextmanager
40def removeWritePermission(filename):
41 """Remove the write permission on a file."""
42 mode = os.stat(filename).st_mode
43 try:
44 os.chmod(filename, stat.S_IREAD)
45 yield
46 finally:
47 os.chmod(filename, mode)
50def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
51 """Check whether we really can modify a database.
53 This intentionally allows any exception to be raised (not just
54 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
55 but the Database was initialized (incorrectly) with writeable=True.
56 """
57 try:
58 with database.declareStaticTables(create=True) as context:
59 table = context.addTable(
60 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
61 )
62 # Drop created table so that schema remains empty.
63 database._metadata.drop_all(database._engine, tables=[table])
64 return True
65 except Exception:
66 return False
69class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
70 """Tests for `SqliteDatabase` using a standard file-based database."""
72 def setUp(self):
73 self.root = makeTestTempDir(TESTDIR)
75 def tearDown(self):
76 removeTestTempDir(self.root)
78 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
79 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
80 engine = SqliteDatabase.makeEngine(filename=filename)
81 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
83 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
84 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable)
85 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable)
87 @contextmanager
88 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
89 with removeWritePermission(database.filename):
90 yield self.getNewConnection(database, writeable=False)
92 def testConnection(self):
93 """Test that different ways of connecting to a SQLite database
94 are equivalent.
95 """
96 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
97 # Create a read-write database by passing in the filename.
98 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0)
99 self.assertEqual(rwFromFilename.filename, filename)
100 self.assertEqual(rwFromFilename.origin, 0)
101 self.assertTrue(rwFromFilename.isWriteable())
102 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
103 # Create a read-write database via a URI.
104 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
105 self.assertEqual(rwFromUri.filename, filename)
106 self.assertEqual(rwFromUri.origin, 0)
107 self.assertTrue(rwFromUri.isWriteable())
108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
109 # We don't support SQLite URIs inside SQLAlchemy URIs.
110 with self.assertRaises(NotImplementedError):
111 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true")
113 # Test read-only connections against a read-only file.
114 with removeWritePermission(filename):
115 # Create a read-only database by passing in the filename.
116 roFromFilename = SqliteDatabase.fromEngine(
117 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False
118 )
119 self.assertEqual(roFromFilename.filename, filename)
120 self.assertEqual(roFromFilename.origin, 0)
121 self.assertFalse(roFromFilename.isWriteable())
122 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
123 # Create a read-write database via a URI.
124 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
125 self.assertEqual(roFromUri.filename, filename)
126 self.assertEqual(roFromUri.origin, 0)
127 self.assertFalse(roFromUri.isWriteable())
128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
130 def testTransactionLocking(self):
131 # This (inherited) test can't run on SQLite because of our use of an
132 # aggressive locking strategy there.
133 pass
136class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
137 """Tests for `SqliteDatabase` using an in-memory database."""
139 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
140 engine = SqliteDatabase.makeEngine(filename=None)
141 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
143 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
144 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable)
146 @contextmanager
147 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
148 yield self.getNewConnection(database, writeable=False)
150 def testConnection(self):
151 """Test that different ways of connecting to a SQLite database
152 are equivalent.
153 """
154 # Create an in-memory database by passing filename=None.
155 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0)
156 self.assertIsNone(memFromFilename.filename)
157 self.assertEqual(memFromFilename.origin, 0)
158 self.assertTrue(memFromFilename.isWriteable())
159 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
160 # Create an in-memory database via a URI.
161 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
162 self.assertIsNone(memFromUri.filename)
163 self.assertEqual(memFromUri.origin, 0)
164 self.assertTrue(memFromUri.isWriteable())
165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
166 # We don't support SQLite URIs inside SQLAlchemy URIs.
167 with self.assertRaises(NotImplementedError):
168 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true")
169 # We don't support read-only in-memory databases.
170 with self.assertRaises(NotImplementedError):
171 SqliteDatabase.makeEngine(filename=None, writeable=False)
173 def testTransactionLocking(self):
174 # This (inherited) test can't run on SQLite because of our use of an
175 # aggressive locking strategy there.
176 pass
179class SqliteFileRegistryTests(RegistryTests):
180 """Tests for `Registry` backed by a SQLite file-based database.
182 Notes
183 -----
184 This is not a subclass of `unittest.TestCase` but to avoid repetition it
185 defines methods that override `unittest.TestCase` methods. To make this
186 work sublasses have to have this class first in the bases list.
187 """
189 def setUp(self):
190 self.root = makeTestTempDir(TESTDIR)
192 def tearDown(self):
193 removeTestTempDir(self.root)
195 @classmethod
196 def getDataDir(cls) -> str:
197 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
199 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry:
200 if share_repo_with is None:
201 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
202 else:
203 filename = share_repo_with._db.filename
204 config = self.makeRegistryConfig()
205 config["db"] = f"sqlite:///{filename}"
206 if share_repo_with is None:
207 return _RegistryFactory(config).create_from_config(butlerRoot=self.root)
208 else:
209 return _RegistryFactory(config).from_config(butlerRoot=self.root)
212class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
213 """Tests for `Registry` backed by a SQLite file-based database.
215 This test case uses NameKeyCollectionManager and
216 ByDimensionsDatasetRecordStorageManagerUUID.
217 """
219 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
220 datasetsManager = (
221 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
222 )
225class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
226 """Tests for `Registry` backed by a SQLite file-based database.
228 This test case uses SynthIntKeyCollectionManager and
229 ByDimensionsDatasetRecordStorageManagerUUID.
230 """
232 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
233 datasetsManager = (
234 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
235 )
238class SqliteMemoryRegistryTests(RegistryTests):
239 """Tests for `Registry` backed by a SQLite in-memory database."""
241 @classmethod
242 def getDataDir(cls) -> str:
243 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
245 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry | None:
246 if share_repo_with is not None:
247 return None
248 config = self.makeRegistryConfig()
249 config["db"] = "sqlite://"
250 return _RegistryFactory(config).create_from_config()
252 def testMissingAttributes(self):
253 """Test for instantiating a registry against outdated schema which
254 misses butler_attributes table.
255 """
256 # TODO: Once we have stable gen3 schema everywhere this test can be
257 # dropped (DM-27373).
258 config = self.makeRegistryConfig()
259 config["db"] = "sqlite://"
260 with self.assertRaises(LookupError):
261 _RegistryFactory(config).from_config()
264class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
265 """Tests for `Registry` backed by a SQLite in-memory database.
267 This test case uses NameKeyCollectionManager and
268 ByDimensionsDatasetRecordStorageManagerUUID.
269 """
271 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
272 datasetsManager = (
273 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
274 )
277class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
278 """Tests for `Registry` backed by a SQLite in-memory database.
280 This test case uses SynthIntKeyCollectionManager and
281 ByDimensionsDatasetRecordStorageManagerUUID.
282 """
284 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
285 datasetsManager = (
286 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
287 )
290class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
291 """Tests for `Registry` backed by a SQLite in-memory database.
293 This test case uses version schema with ingest_date as nanoseconds instead
294 of DATETIME. This tests can be removed when/if we switch to nanoseconds as
295 default.
296 """
298 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
299 datasetsManager = {
300 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID",
301 "schema_version": "2.0.0",
302 }
305if __name__ == "__main__":
306 unittest.main()