Coverage for tests/test_sqlite.py: 39%
145 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import os
29import os.path
30import stat
31import tempfile
32import unittest
33from contextlib import contextmanager
35import sqlalchemy
36from lsst.daf.butler import ddl
37from lsst.daf.butler.registry import _RegistryFactory
38from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
39from lsst.daf.butler.registry.sql_registry import SqlRegistry
40from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
41from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
43TESTDIR = os.path.abspath(os.path.dirname(__file__))
46@contextmanager
47def removeWritePermission(filename):
48 """Remove the write permission on a file."""
49 mode = os.stat(filename).st_mode
50 try:
51 os.chmod(filename, stat.S_IREAD)
52 yield
53 finally:
54 os.chmod(filename, mode)
57def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
58 """Check whether we really can modify a database.
60 This intentionally allows any exception to be raised (not just
61 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
62 but the Database was initialized (incorrectly) with writeable=True.
63 """
64 try:
65 with database.declareStaticTables(create=True) as context:
66 table = context.addTable(
67 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
68 )
69 # Drop created table so that schema remains empty.
70 database._metadata.drop_all(database._engine, tables=[table])
71 return True
72 except Exception:
73 return False
76class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
77 """Tests for `SqliteDatabase` using a standard file-based database."""
79 def setUp(self):
80 self.root = makeTestTempDir(TESTDIR)
82 def tearDown(self):
83 removeTestTempDir(self.root)
85 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
86 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
87 engine = SqliteDatabase.makeEngine(filename=filename)
88 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
90 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
91 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable)
92 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable)
94 @contextmanager
95 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
96 with removeWritePermission(database.filename):
97 yield self.getNewConnection(database, writeable=False)
99 def testConnection(self):
100 """Test that different ways of connecting to a SQLite database
101 are equivalent.
102 """
103 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
104 # Create a read-write database by passing in the filename.
105 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0)
106 self.assertEqual(rwFromFilename.filename, filename)
107 self.assertEqual(rwFromFilename.origin, 0)
108 self.assertTrue(rwFromFilename.isWriteable())
109 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
110 # Create a read-write database via a URI.
111 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
112 self.assertEqual(rwFromUri.filename, filename)
113 self.assertEqual(rwFromUri.origin, 0)
114 self.assertTrue(rwFromUri.isWriteable())
115 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
116 # We don't support SQLite URIs inside SQLAlchemy URIs.
117 with self.assertRaises(NotImplementedError):
118 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true")
120 # Test read-only connections against a read-only file.
121 with removeWritePermission(filename):
122 # Create a read-only database by passing in the filename.
123 roFromFilename = SqliteDatabase.fromEngine(
124 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False
125 )
126 self.assertEqual(roFromFilename.filename, filename)
127 self.assertEqual(roFromFilename.origin, 0)
128 self.assertFalse(roFromFilename.isWriteable())
129 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
130 # Create a read-write database via a URI.
131 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
132 self.assertEqual(roFromUri.filename, filename)
133 self.assertEqual(roFromUri.origin, 0)
134 self.assertFalse(roFromUri.isWriteable())
135 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
137 def testTransactionLocking(self):
138 # This (inherited) test can't run on SQLite because of our use of an
139 # aggressive locking strategy there.
140 pass
143class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
144 """Tests for `SqliteDatabase` using an in-memory database."""
146 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
147 engine = SqliteDatabase.makeEngine(filename=None)
148 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
150 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
151 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable)
153 @contextmanager
154 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
155 yield self.getNewConnection(database, writeable=False)
157 def testConnection(self):
158 """Test that different ways of connecting to a SQLite database
159 are equivalent.
160 """
161 # Create an in-memory database by passing filename=None.
162 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0)
163 self.assertIsNone(memFromFilename.filename)
164 self.assertEqual(memFromFilename.origin, 0)
165 self.assertTrue(memFromFilename.isWriteable())
166 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
167 # Create an in-memory database via a URI.
168 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
169 self.assertIsNone(memFromUri.filename)
170 self.assertEqual(memFromUri.origin, 0)
171 self.assertTrue(memFromUri.isWriteable())
172 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
173 # We don't support SQLite URIs inside SQLAlchemy URIs.
174 with self.assertRaises(NotImplementedError):
175 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true")
176 # We don't support read-only in-memory databases.
177 with self.assertRaises(NotImplementedError):
178 SqliteDatabase.makeEngine(filename=None, writeable=False)
180 def testTransactionLocking(self):
181 # This (inherited) test can't run on SQLite because of our use of an
182 # aggressive locking strategy there.
183 pass
186class SqliteFileRegistryTests(RegistryTests):
187 """Tests for `Registry` backed by a SQLite file-based database.
189 Notes
190 -----
191 This is not a subclass of `unittest.TestCase` but to avoid repetition it
192 defines methods that override `unittest.TestCase` methods. To make this
193 work sublasses have to have this class first in the bases list.
194 """
196 def setUp(self):
197 self.root = makeTestTempDir(TESTDIR)
199 def tearDown(self):
200 removeTestTempDir(self.root)
202 @classmethod
203 def getDataDir(cls) -> str:
204 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
206 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry:
207 if share_repo_with is None:
208 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
209 else:
210 filename = share_repo_with._db.filename
211 config = self.makeRegistryConfig()
212 config["db"] = f"sqlite:///{filename}"
213 if share_repo_with is None:
214 return _RegistryFactory(config).create_from_config(butlerRoot=self.root)
215 else:
216 return _RegistryFactory(config).from_config(butlerRoot=self.root)
219class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
220 """Tests for `Registry` backed by a SQLite file-based database.
222 This test case uses NameKeyCollectionManager and
223 ByDimensionsDatasetRecordStorageManagerUUID.
224 """
226 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
227 datasetsManager = (
228 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
229 )
232class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
233 """Tests for `Registry` backed by a SQLite file-based database.
235 This test case uses SynthIntKeyCollectionManager and
236 ByDimensionsDatasetRecordStorageManagerUUID.
237 """
239 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
240 datasetsManager = (
241 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
242 )
245class SqliteMemoryRegistryTests(RegistryTests):
246 """Tests for `Registry` backed by a SQLite in-memory database."""
248 @classmethod
249 def getDataDir(cls) -> str:
250 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
252 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry | None:
253 if share_repo_with is not None:
254 return None
255 config = self.makeRegistryConfig()
256 config["db"] = "sqlite://"
257 return _RegistryFactory(config).create_from_config()
259 def testMissingAttributes(self):
260 """Test for instantiating a registry against outdated schema which
261 misses butler_attributes table.
262 """
263 # TODO: Once we have stable gen3 schema everywhere this test can be
264 # dropped (DM-27373).
265 config = self.makeRegistryConfig()
266 config["db"] = "sqlite://"
267 with self.assertRaises(LookupError):
268 _RegistryFactory(config).from_config()
271class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
272 """Tests for `Registry` backed by a SQLite in-memory database.
274 This test case uses NameKeyCollectionManager and
275 ByDimensionsDatasetRecordStorageManagerUUID.
276 """
278 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
279 datasetsManager = (
280 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
281 )
284class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
285 """Tests for `Registry` backed by a SQLite in-memory database.
287 This test case uses SynthIntKeyCollectionManager and
288 ByDimensionsDatasetRecordStorageManagerUUID.
289 """
291 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
292 datasetsManager = (
293 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
294 )
297class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
298 """Tests for `Registry` backed by a SQLite in-memory database.
300 This test case uses version schema with ingest_date as nanoseconds instead
301 of DATETIME. This tests can be removed when/if we switch to nanoseconds as
302 default.
303 """
305 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
306 datasetsManager = {
307 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID",
308 "schema_version": "2.0.0",
309 }
312if __name__ == "__main__":
313 unittest.main()