Coverage for tests/test_sqlite.py: 40%
149 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-07 11:04 +0000
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-07 11:04 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import os
29import os.path
30import stat
31import tempfile
32import unittest
33from contextlib import contextmanager
35import sqlalchemy
36from lsst.daf.butler import ddl
37from lsst.daf.butler.registry import _RegistryFactory
38from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
39from lsst.daf.butler.registry.sql_registry import SqlRegistry
40from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
41from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
43TESTDIR = os.path.abspath(os.path.dirname(__file__))
46@contextmanager
47def removeWritePermission(filename):
48 """Remove the write permission on a file."""
49 mode = os.stat(filename).st_mode
50 try:
51 os.chmod(filename, stat.S_IREAD)
52 yield
53 finally:
54 os.chmod(filename, mode)
57def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
58 """Check whether we really can modify a database.
60 This intentionally allows any exception to be raised (not just
61 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
62 but the Database was initialized (incorrectly) with writeable=True.
63 """
64 try:
65 with database.declareStaticTables(create=True) as context:
66 table = context.addTable(
67 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
68 )
69 # Drop created table so that schema remains empty.
70 database._metadata.drop_all(database._engine, tables=[table])
71 return True
72 except Exception:
73 return False
76class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
77 """Tests for `SqliteDatabase` using a standard file-based database."""
79 def setUp(self):
80 self.root = makeTestTempDir(TESTDIR)
82 def tearDown(self):
83 removeTestTempDir(self.root)
85 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
86 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
87 engine = SqliteDatabase.makeEngine(filename=filename)
88 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
90 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
91 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable)
92 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable)
94 @contextmanager
95 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
96 with removeWritePermission(database.filename):
97 yield self.getNewConnection(database, writeable=False)
99 def testConnection(self):
100 """Test that different ways of connecting to a SQLite database
101 are equivalent.
102 """
103 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
104 # Create a read-write database by passing in the filename.
105 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0)
106 self.assertEqual(rwFromFilename.filename, filename)
107 self.assertEqual(rwFromFilename.origin, 0)
108 self.assertTrue(rwFromFilename.isWriteable())
109 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
110 # Create a read-write database via a URI.
111 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
112 self.assertEqual(rwFromUri.filename, filename)
113 self.assertEqual(rwFromUri.origin, 0)
114 self.assertTrue(rwFromUri.isWriteable())
115 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
116 # We don't support SQLite URIs inside SQLAlchemy URIs.
117 with self.assertRaises(NotImplementedError):
118 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true")
120 # Test read-only connections against a read-only file.
121 with removeWritePermission(filename):
122 # Create a read-only database by passing in the filename.
123 roFromFilename = SqliteDatabase.fromEngine(
124 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False
125 )
126 self.assertEqual(roFromFilename.filename, filename)
127 self.assertEqual(roFromFilename.origin, 0)
128 self.assertFalse(roFromFilename.isWriteable())
129 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
130 # Create a read-write database via a URI.
131 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
132 self.assertEqual(roFromUri.filename, filename)
133 self.assertEqual(roFromUri.origin, 0)
134 self.assertFalse(roFromUri.isWriteable())
135 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
137 def testTransactionLocking(self):
138 # This (inherited) test can't run on SQLite because of our use of an
139 # aggressive locking strategy there.
140 pass
143class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
144 """Tests for `SqliteDatabase` using an in-memory database."""
146 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
147 engine = SqliteDatabase.makeEngine(filename=None)
148 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
150 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
151 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable)
153 @contextmanager
154 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
155 yield self.getNewConnection(database, writeable=False)
157 def testConnection(self):
158 """Test that different ways of connecting to a SQLite database
159 are equivalent.
160 """
161 # Create an in-memory database by passing filename=None.
162 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0)
163 self.assertIsNone(memFromFilename.filename)
164 self.assertEqual(memFromFilename.origin, 0)
165 self.assertTrue(memFromFilename.isWriteable())
166 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
167 # Create an in-memory database via a URI.
168 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
169 self.assertIsNone(memFromUri.filename)
170 self.assertEqual(memFromUri.origin, 0)
171 self.assertTrue(memFromUri.isWriteable())
172 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
173 # We don't support SQLite URIs inside SQLAlchemy URIs.
174 with self.assertRaises(NotImplementedError):
175 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true")
176 # We don't support read-only in-memory databases.
177 with self.assertRaises(NotImplementedError):
178 SqliteDatabase.makeEngine(filename=None, writeable=False)
180 def testTransactionLocking(self):
181 # This (inherited) test can't run on SQLite because of our use of an
182 # aggressive locking strategy there.
183 pass
186class SqliteFileRegistryTests(RegistryTests):
187 """Tests for `Registry` backed by a SQLite file-based database.
189 Notes
190 -----
191 This is not a subclass of `unittest.TestCase` but to avoid repetition it
192 defines methods that override `unittest.TestCase` methods. To make this
193 work sublasses have to have this class first in the bases list.
194 """
196 def setUp(self):
197 self.root = makeTestTempDir(TESTDIR)
199 def tearDown(self):
200 removeTestTempDir(self.root)
202 @classmethod
203 def getDataDir(cls) -> str:
204 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
206 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry:
207 if share_repo_with is None:
208 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
209 else:
210 filename = share_repo_with._db.filename
211 config = self.makeRegistryConfig()
212 config["db"] = f"sqlite:///{filename}"
213 if share_repo_with is None:
214 return _RegistryFactory(config).create_from_config(butlerRoot=self.root)
215 else:
216 return _RegistryFactory(config).from_config(butlerRoot=self.root)
219class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
220 """Tests for `Registry` backed by a SQLite file-based database.
222 This test case uses NameKeyCollectionManager and
223 ByDimensionsDatasetRecordStorageManagerUUID.
224 """
226 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
227 datasetsManager = (
228 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
229 )
232class ClonedSqliteFileRegistryNameKeyCollMgrUUIDTestCase(
233 SqliteFileRegistryNameKeyCollMgrUUIDTestCase, unittest.TestCase
234):
235 """Test that NameKeyCollectionManager still works after cloning."""
237 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry:
238 original = super().makeRegistry(share_repo_with)
239 return original.copy()
242class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
243 """Tests for `Registry` backed by a SQLite file-based database.
245 This test case uses SynthIntKeyCollectionManager and
246 ByDimensionsDatasetRecordStorageManagerUUID.
247 """
249 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
250 datasetsManager = (
251 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
252 )
255class SqliteMemoryRegistryTests(RegistryTests):
256 """Tests for `Registry` backed by a SQLite in-memory database."""
258 @classmethod
259 def getDataDir(cls) -> str:
260 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
262 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry | None:
263 if share_repo_with is not None:
264 return None
265 config = self.makeRegistryConfig()
266 config["db"] = "sqlite://"
267 return _RegistryFactory(config).create_from_config()
269 def testMissingAttributes(self):
270 """Test for instantiating a registry against outdated schema which
271 misses butler_attributes table.
272 """
273 # TODO: Once we have stable gen3 schema everywhere this test can be
274 # dropped (DM-27373).
275 config = self.makeRegistryConfig()
276 config["db"] = "sqlite://"
277 with self.assertRaises(LookupError):
278 _RegistryFactory(config).from_config()
281class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
282 """Tests for `Registry` backed by a SQLite in-memory database.
284 This test case uses NameKeyCollectionManager and
285 ByDimensionsDatasetRecordStorageManagerUUID.
286 """
288 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
289 datasetsManager = (
290 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
291 )
294class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
295 """Tests for `Registry` backed by a SQLite in-memory database.
297 This test case uses SynthIntKeyCollectionManager and
298 ByDimensionsDatasetRecordStorageManagerUUID.
299 """
301 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
302 datasetsManager = (
303 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
304 )
307class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
308 """Tests for `Registry` backed by a SQLite in-memory database.
310 This test case uses version schema with ingest_date as nanoseconds instead
311 of DATETIME. This tests can be removed when/if we switch to nanoseconds as
312 default.
313 """
315 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
316 datasetsManager = {
317 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID",
318 "schema_version": "2.0.0",
319 }
322if __name__ == "__main__":
323 unittest.main()