Coverage for tests/test_sqlite.py: 39%
144 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import os
29import os.path
30import stat
31import tempfile
32import unittest
33from contextlib import contextmanager
35import sqlalchemy
36from lsst.daf.butler import ddl
37from lsst.daf.butler.registry import _ButlerRegistry, _RegistryFactory
38from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
39from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
40from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
42TESTDIR = os.path.abspath(os.path.dirname(__file__))
45@contextmanager
46def removeWritePermission(filename):
47 """Remove the write permission on a file."""
48 mode = os.stat(filename).st_mode
49 try:
50 os.chmod(filename, stat.S_IREAD)
51 yield
52 finally:
53 os.chmod(filename, mode)
56def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
57 """Check whether we really can modify a database.
59 This intentionally allows any exception to be raised (not just
60 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
61 but the Database was initialized (incorrectly) with writeable=True.
62 """
63 try:
64 with database.declareStaticTables(create=True) as context:
65 table = context.addTable(
66 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
67 )
68 # Drop created table so that schema remains empty.
69 database._metadata.drop_all(database._engine, tables=[table])
70 return True
71 except Exception:
72 return False
75class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
76 """Tests for `SqliteDatabase` using a standard file-based database."""
78 def setUp(self):
79 self.root = makeTestTempDir(TESTDIR)
81 def tearDown(self):
82 removeTestTempDir(self.root)
84 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
85 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
86 engine = SqliteDatabase.makeEngine(filename=filename)
87 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
89 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
90 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable)
91 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable)
93 @contextmanager
94 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
95 with removeWritePermission(database.filename):
96 yield self.getNewConnection(database, writeable=False)
98 def testConnection(self):
99 """Test that different ways of connecting to a SQLite database
100 are equivalent.
101 """
102 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
103 # Create a read-write database by passing in the filename.
104 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0)
105 self.assertEqual(rwFromFilename.filename, filename)
106 self.assertEqual(rwFromFilename.origin, 0)
107 self.assertTrue(rwFromFilename.isWriteable())
108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
109 # Create a read-write database via a URI.
110 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
111 self.assertEqual(rwFromUri.filename, filename)
112 self.assertEqual(rwFromUri.origin, 0)
113 self.assertTrue(rwFromUri.isWriteable())
114 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
115 # We don't support SQLite URIs inside SQLAlchemy URIs.
116 with self.assertRaises(NotImplementedError):
117 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true")
119 # Test read-only connections against a read-only file.
120 with removeWritePermission(filename):
121 # Create a read-only database by passing in the filename.
122 roFromFilename = SqliteDatabase.fromEngine(
123 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False
124 )
125 self.assertEqual(roFromFilename.filename, filename)
126 self.assertEqual(roFromFilename.origin, 0)
127 self.assertFalse(roFromFilename.isWriteable())
128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
129 # Create a read-write database via a URI.
130 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
131 self.assertEqual(roFromUri.filename, filename)
132 self.assertEqual(roFromUri.origin, 0)
133 self.assertFalse(roFromUri.isWriteable())
134 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
136 def testTransactionLocking(self):
137 # This (inherited) test can't run on SQLite because of our use of an
138 # aggressive locking strategy there.
139 pass
142class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
143 """Tests for `SqliteDatabase` using an in-memory database."""
145 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
146 engine = SqliteDatabase.makeEngine(filename=None)
147 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
149 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
150 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable)
152 @contextmanager
153 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
154 yield self.getNewConnection(database, writeable=False)
156 def testConnection(self):
157 """Test that different ways of connecting to a SQLite database
158 are equivalent.
159 """
160 # Create an in-memory database by passing filename=None.
161 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0)
162 self.assertIsNone(memFromFilename.filename)
163 self.assertEqual(memFromFilename.origin, 0)
164 self.assertTrue(memFromFilename.isWriteable())
165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
166 # Create an in-memory database via a URI.
167 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
168 self.assertIsNone(memFromUri.filename)
169 self.assertEqual(memFromUri.origin, 0)
170 self.assertTrue(memFromUri.isWriteable())
171 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
172 # We don't support SQLite URIs inside SQLAlchemy URIs.
173 with self.assertRaises(NotImplementedError):
174 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true")
175 # We don't support read-only in-memory databases.
176 with self.assertRaises(NotImplementedError):
177 SqliteDatabase.makeEngine(filename=None, writeable=False)
179 def testTransactionLocking(self):
180 # This (inherited) test can't run on SQLite because of our use of an
181 # aggressive locking strategy there.
182 pass
185class SqliteFileRegistryTests(RegistryTests):
186 """Tests for `Registry` backed by a SQLite file-based database.
188 Notes
189 -----
190 This is not a subclass of `unittest.TestCase` but to avoid repetition it
191 defines methods that override `unittest.TestCase` methods. To make this
192 work sublasses have to have this class first in the bases list.
193 """
195 def setUp(self):
196 self.root = makeTestTempDir(TESTDIR)
198 def tearDown(self):
199 removeTestTempDir(self.root)
201 @classmethod
202 def getDataDir(cls) -> str:
203 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
205 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry:
206 if share_repo_with is None:
207 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
208 else:
209 filename = share_repo_with._db.filename
210 config = self.makeRegistryConfig()
211 config["db"] = f"sqlite:///{filename}"
212 if share_repo_with is None:
213 return _RegistryFactory(config).create_from_config(butlerRoot=self.root)
214 else:
215 return _RegistryFactory(config).from_config(butlerRoot=self.root)
218class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
219 """Tests for `Registry` backed by a SQLite file-based database.
221 This test case uses NameKeyCollectionManager and
222 ByDimensionsDatasetRecordStorageManagerUUID.
223 """
225 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
226 datasetsManager = (
227 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
228 )
231class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
232 """Tests for `Registry` backed by a SQLite file-based database.
234 This test case uses SynthIntKeyCollectionManager and
235 ByDimensionsDatasetRecordStorageManagerUUID.
236 """
238 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
239 datasetsManager = (
240 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
241 )
244class SqliteMemoryRegistryTests(RegistryTests):
245 """Tests for `Registry` backed by a SQLite in-memory database."""
247 @classmethod
248 def getDataDir(cls) -> str:
249 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
251 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry | None:
252 if share_repo_with is not None:
253 return None
254 config = self.makeRegistryConfig()
255 config["db"] = "sqlite://"
256 return _RegistryFactory(config).create_from_config()
258 def testMissingAttributes(self):
259 """Test for instantiating a registry against outdated schema which
260 misses butler_attributes table.
261 """
262 # TODO: Once we have stable gen3 schema everywhere this test can be
263 # dropped (DM-27373).
264 config = self.makeRegistryConfig()
265 config["db"] = "sqlite://"
266 with self.assertRaises(LookupError):
267 _RegistryFactory(config).from_config()
270class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
271 """Tests for `Registry` backed by a SQLite in-memory database.
273 This test case uses NameKeyCollectionManager and
274 ByDimensionsDatasetRecordStorageManagerUUID.
275 """
277 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
278 datasetsManager = (
279 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
280 )
283class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
284 """Tests for `Registry` backed by a SQLite in-memory database.
286 This test case uses SynthIntKeyCollectionManager and
287 ByDimensionsDatasetRecordStorageManagerUUID.
288 """
290 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
291 datasetsManager = (
292 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
293 )
296class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
297 """Tests for `Registry` backed by a SQLite in-memory database.
299 This test case uses version schema with ingest_date as nanoseconds instead
300 of DATETIME. This tests can be removed when/if we switch to nanoseconds as
301 default.
302 """
304 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
305 datasetsManager = {
306 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID",
307 "schema_version": "2.0.0",
308 }
311if __name__ == "__main__":
312 unittest.main()