Coverage for tests/test_sqlite.py: 35%
144 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 09:30 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 09:30 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import os.path
24import stat
25import tempfile
26import unittest
27from contextlib import contextmanager
29import sqlalchemy
30from lsst.daf.butler import ddl
31from lsst.daf.butler.registry import Registry
32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
33from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
34from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
36TESTDIR = os.path.abspath(os.path.dirname(__file__))
39@contextmanager
40def removeWritePermission(filename):
41 mode = os.stat(filename).st_mode
42 try:
43 os.chmod(filename, stat.S_IREAD)
44 yield
45 finally:
46 os.chmod(filename, mode)
49def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
50 """Check whether we really can modify a database.
52 This intentionally allows any exception to be raised (not just
53 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
54 but the Database was initialized (incorrectly) with writeable=True.
55 """
56 try:
57 with database.declareStaticTables(create=True) as context:
58 table = context.addTable(
59 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
60 )
61 # Drop created table so that schema remains empty.
62 database._metadata.drop_all(database._engine, tables=[table])
63 return True
64 except Exception:
65 return False
68class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
69 """Tests for `SqliteDatabase` using a standard file-based database."""
71 def setUp(self):
72 self.root = makeTestTempDir(TESTDIR)
74 def tearDown(self):
75 removeTestTempDir(self.root)
77 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
78 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
79 engine = SqliteDatabase.makeEngine(filename=filename)
80 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
82 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
83 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable)
84 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable)
86 @contextmanager
87 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
88 with removeWritePermission(database.filename):
89 yield self.getNewConnection(database, writeable=False)
91 def testConnection(self):
92 """Test that different ways of connecting to a SQLite database
93 are equivalent.
94 """
95 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
96 # Create a read-write database by passing in the filename.
97 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0)
98 self.assertEqual(rwFromFilename.filename, filename)
99 self.assertEqual(rwFromFilename.origin, 0)
100 self.assertTrue(rwFromFilename.isWriteable())
101 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
102 # Create a read-write database via a URI.
103 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
104 self.assertEqual(rwFromUri.filename, filename)
105 self.assertEqual(rwFromUri.origin, 0)
106 self.assertTrue(rwFromUri.isWriteable())
107 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
108 # We don't support SQLite URIs inside SQLAlchemy URIs.
109 with self.assertRaises(NotImplementedError):
110 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true")
112 # Test read-only connections against a read-only file.
113 with removeWritePermission(filename):
114 # Create a read-only database by passing in the filename.
115 roFromFilename = SqliteDatabase.fromEngine(
116 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False
117 )
118 self.assertEqual(roFromFilename.filename, filename)
119 self.assertEqual(roFromFilename.origin, 0)
120 self.assertFalse(roFromFilename.isWriteable())
121 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
122 # Create a read-write database via a URI.
123 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
124 self.assertEqual(roFromUri.filename, filename)
125 self.assertEqual(roFromUri.origin, 0)
126 self.assertFalse(roFromUri.isWriteable())
127 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
129 def testTransactionLocking(self):
130 # This (inherited) test can't run on SQLite because of our use of an
131 # aggressive locking strategy there.
132 pass
135class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
136 """Tests for `SqliteDatabase` using an in-memory database."""
138 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
139 engine = SqliteDatabase.makeEngine(filename=None)
140 return SqliteDatabase.fromEngine(engine=engine, origin=origin)
142 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
143 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable)
145 @contextmanager
146 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
147 yield self.getNewConnection(database, writeable=False)
149 def testConnection(self):
150 """Test that different ways of connecting to a SQLite database
151 are equivalent.
152 """
153 # Create an in-memory database by passing filename=None.
154 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0)
155 self.assertIsNone(memFromFilename.filename)
156 self.assertEqual(memFromFilename.origin, 0)
157 self.assertTrue(memFromFilename.isWriteable())
158 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
159 # Create an in-memory database via a URI.
160 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
161 self.assertIsNone(memFromUri.filename)
162 self.assertEqual(memFromUri.origin, 0)
163 self.assertTrue(memFromUri.isWriteable())
164 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
165 # We don't support SQLite URIs inside SQLAlchemy URIs.
166 with self.assertRaises(NotImplementedError):
167 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true")
168 # We don't support read-only in-memory databases.
169 with self.assertRaises(NotImplementedError):
170 SqliteDatabase.makeEngine(filename=None, writeable=False)
172 def testTransactionLocking(self):
173 # This (inherited) test can't run on SQLite because of our use of an
174 # aggressive locking strategy there.
175 pass
178class SqliteFileRegistryTests(RegistryTests):
179 """Tests for `Registry` backed by a SQLite file-based database.
181 Note
182 ----
183 This is not a subclass of `unittest.TestCase` but to avoid repetition it
184 defines methods that override `unittest.TestCase` methods. To make this
185 work sublasses have to have this class first in the bases list.
186 """
188 def setUp(self):
189 self.root = makeTestTempDir(TESTDIR)
191 def tearDown(self):
192 removeTestTempDir(self.root)
194 @classmethod
195 def getDataDir(cls) -> str:
196 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
198 def makeRegistry(self, share_repo_with: Registry | None = None) -> Registry:
199 if share_repo_with is None:
200 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
201 else:
202 filename = share_repo_with._db.filename
203 config = self.makeRegistryConfig()
204 config["db"] = f"sqlite:///{filename}"
205 if share_repo_with is None:
206 return Registry.createFromConfig(config, butlerRoot=self.root)
207 else:
208 return Registry.fromConfig(config, butlerRoot=self.root)
211class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
212 """Tests for `Registry` backed by a SQLite file-based database.
214 This test case uses NameKeyCollectionManager and
215 ByDimensionsDatasetRecordStorageManagerUUID.
216 """
218 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
219 datasetsManager = (
220 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
221 )
224class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase):
225 """Tests for `Registry` backed by a SQLite file-based database.
227 This test case uses SynthIntKeyCollectionManager and
228 ByDimensionsDatasetRecordStorageManagerUUID.
229 """
231 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
232 datasetsManager = (
233 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
234 )
237class SqliteMemoryRegistryTests(RegistryTests):
238 """Tests for `Registry` backed by a SQLite in-memory database."""
240 @classmethod
241 def getDataDir(cls) -> str:
242 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
244 def makeRegistry(self, share_repo_with: Registry | None = None) -> Registry | None:
245 if share_repo_with is not None:
246 return None
247 config = self.makeRegistryConfig()
248 config["db"] = "sqlite://"
249 return Registry.createFromConfig(config)
251 def testMissingAttributes(self):
252 """Test for instantiating a registry against outdated schema which
253 misses butler_attributes table.
254 """
255 # TODO: Once we have stable gen3 schema everywhere this test can be
256 # dropped (DM-27373).
257 config = self.makeRegistryConfig()
258 config["db"] = "sqlite://"
259 with self.assertRaises(LookupError):
260 Registry.fromConfig(config)
263class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
264 """Tests for `Registry` backed by a SQLite in-memory database.
266 This test case uses NameKeyCollectionManager and
267 ByDimensionsDatasetRecordStorageManagerUUID.
268 """
270 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
271 datasetsManager = (
272 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
273 )
276class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
277 """Tests for `Registry` backed by a SQLite in-memory database.
279 This test case uses SynthIntKeyCollectionManager and
280 ByDimensionsDatasetRecordStorageManagerUUID.
281 """
283 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
284 datasetsManager = (
285 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID"
286 )
289class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
290 """Tests for `Registry` backed by a SQLite in-memory database.
292 This test case uses version schema with ingest_date as nanoseconds instead
293 of DATETIME. This tests can be removed when/if we switch to nanoseconds as
294 default.
295 """
297 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
298 datasetsManager = {
299 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID",
300 "schema_version": "2.0.0",
301 }
304if __name__ == "__main__":
305 unittest.main()