Coverage for tests/test_sqlite.py : 31%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from contextlib import contextmanager
23import itertools
24import os
25import os.path
26import shutil
27import stat
28import tempfile
29import unittest
31import sqlalchemy
33from lsst.sphgeom import ConvexPolygon, UnitVector3d
35from lsst.daf.butler import ddl
36from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
37from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
38from lsst.daf.butler.registry import Registry
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
43@contextmanager
44def removeWritePermission(filename):
45 mode = os.stat(filename).st_mode
46 try:
47 os.chmod(filename, stat.S_IREAD)
48 yield
49 finally:
50 os.chmod(filename, mode)
53def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
54 """Check whether we really can modify a database.
56 This intentionally allows any exception to be raised (not just
57 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
58 but the Database was initialized (incorrectly) with writeable=True.
59 """
60 try:
61 with database.declareStaticTables(create=True) as context:
62 table = context.addTable(
63 "a",
64 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
65 )
66 # Drop created table so that schema remains empty.
67 database._metadata.drop_all(database._connection, tables=[table])
68 return True
69 except Exception:
70 return False
73class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
74 """Tests for `SqliteDatabase` using a standard file-based database.
75 """
77 def setUp(self):
78 self.root = tempfile.mkdtemp(dir=TESTDIR)
80 def tearDown(self):
81 if self.root is not None and os.path.exists(self.root):
82 shutil.rmtree(self.root, ignore_errors=True)
84 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
85 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
86 connection = SqliteDatabase.connect(filename=filename)
87 return SqliteDatabase.fromConnection(connection=connection, origin=origin)
89 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
90 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable)
91 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection,
92 writeable=writeable)
94 @contextmanager
95 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
96 with removeWritePermission(database.filename):
97 yield self.getNewConnection(database, writeable=False)
99 def testConnection(self):
100 """Test that different ways of connecting to a SQLite database
101 are equivalent.
102 """
103 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
104 # Create a read-write database by passing in the filename.
105 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0)
106 self.assertEqual(rwFromFilename.filename, filename)
107 self.assertEqual(rwFromFilename.origin, 0)
108 self.assertTrue(rwFromFilename.isWriteable())
109 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
110 # Create a read-write database via a URI.
111 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
112 self.assertEqual(rwFromUri.filename, filename)
113 self.assertEqual(rwFromUri.origin, 0)
114 self.assertTrue(rwFromUri.isWriteable())
115 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
116 # We don't support SQLite URIs inside SQLAlchemy URIs.
117 with self.assertRaises(NotImplementedError):
118 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true")
120 # Test read-only connections against a read-only file.
121 with removeWritePermission(filename):
122 # Create a read-only database by passing in the filename.
123 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename),
124 origin=0, writeable=False)
125 self.assertEqual(roFromFilename.filename, filename)
126 self.assertEqual(roFromFilename.origin, 0)
127 self.assertFalse(roFromFilename.isWriteable())
128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
129 # Create a read-write database via a URI.
130 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
131 self.assertEqual(roFromUri.filename, filename)
132 self.assertEqual(roFromUri.origin, 0)
133 self.assertFalse(roFromUri.isWriteable())
134 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
136 def testTransactionLocking(self):
137 # This (inherited) test can't run on SQLite because of our use of an
138 # aggressive locking strategy there.
139 pass
142class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
143 """Tests for `SqliteDatabase` using an in-memory database.
144 """
146 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
147 connection = SqliteDatabase.connect(filename=None)
148 return SqliteDatabase.fromConnection(connection=connection, origin=origin)
150 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
151 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection,
152 writeable=writeable)
154 @contextmanager
155 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
156 yield self.getNewConnection(database, writeable=False)
158 def testConnection(self):
159 """Test that different ways of connecting to a SQLite database
160 are equivalent.
161 """
162 # Create an in-memory database by passing filename=None.
163 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0)
164 self.assertIsNone(memFromFilename.filename)
165 self.assertEqual(memFromFilename.origin, 0)
166 self.assertTrue(memFromFilename.isWriteable())
167 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
168 # Create an in-memory database via a URI.
169 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
170 self.assertIsNone(memFromUri.filename)
171 self.assertEqual(memFromUri.origin, 0)
172 self.assertTrue(memFromUri.isWriteable())
173 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
174 # We don't support SQLite URIs inside SQLAlchemy URIs.
175 with self.assertRaises(NotImplementedError):
176 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true")
177 # We don't support read-only in-memory databases.
178 with self.assertRaises(NotImplementedError):
179 SqliteDatabase.connect(filename=None, writeable=False)
181 def testTransactionLocking(self):
182 # This (inherited) test can't run on SQLite because of our use of an
183 # aggressive locking strategy there.
184 pass
187class SqliteFileRegistryTests(RegistryTests):
188 """Tests for `Registry` backed by a SQLite file-based database.
190 Note
191 ----
192 This is not a subclass of `unittest.TestCase` but to avoid repetition it
193 defines methods that override `unittest.TestCase` methods. To make this
194 work sublasses have to have this class first in the bases list.
195 """
197 def setUp(self):
198 self.root = tempfile.mkdtemp(dir=TESTDIR)
200 def tearDown(self):
201 if self.root is not None and os.path.exists(self.root):
202 shutil.rmtree(self.root, ignore_errors=True)
204 @classmethod
205 def getDataDir(cls) -> str:
206 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
208 def makeRegistry(self) -> Registry:
209 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
210 config = self.makeRegistryConfig()
211 config["db"] = f"sqlite:///{filename}"
212 return Registry.fromConfig(config, create=True, butlerRoot=self.root)
215class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase):
216 """Tests for `Registry` backed by a SQLite file-based database.
218 This test case uses NameKeyCollectionManager.
219 """
220 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
223class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase):
224 """Tests for `Registry` backed by a SQLite file-based database.
226 This test case uses SynthIntKeyCollectionManager.
227 """
228 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
231class SqliteMemoryRegistryTests(RegistryTests):
232 """Tests for `Registry` backed by a SQLite in-memory database.
233 """
235 @classmethod
236 def getDataDir(cls) -> str:
237 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
239 def makeRegistry(self) -> Registry:
240 config = self.makeRegistryConfig()
241 config["db"] = "sqlite://"
242 return Registry.fromConfig(config, create=True)
244 def testRegions(self):
245 """Tests for using region fields in `Registry` dimensions.
246 """
247 # TODO: the test regions used here are enormous (significant fractions
248 # of the sphere), and that makes this test prohibitively slow on
249 # most real databases. These should be made more realistic, and the
250 # test moved to daf/butler/registry/tests/registry.py.
251 registry = self.makeRegistry()
252 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0),
253 UnitVector3d(0, 1, 0),
254 UnitVector3d(0, 0, 1)))
255 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0),
256 UnitVector3d(0, 1, 0),
257 UnitVector3d(0, 0, 1)))
258 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0),
259 UnitVector3d(0, 1, 1),
260 UnitVector3d(0, 0, 1)))
261 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0),
262 UnitVector3d(0, 1, 0),
263 UnitVector3d(0, 1, 1)))
264 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2):
265 self.assertNotEqual(a, b)
267 # This depends on current dimensions.yaml definitions
268 self.assertEqual(len(list(registry.queryDataIds(["patch", "htm7"]))), 0)
270 # Add some dimension entries
271 registry.insertDimensionData("instrument", {"name": "DummyCam"})
272 registry.insertDimensionData("physical_filter",
273 {"instrument": "DummyCam", "name": "dummy_r", "band": "r"},
274 {"instrument": "DummyCam", "name": "dummy_i", "band": "i"})
275 for detector in (1, 2, 3, 4, 5):
276 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector,
277 "full_name": str(detector)})
278 registry.insertDimensionData("visit",
279 {"instrument": "DummyCam", "id": 0, "name": "zero",
280 "physical_filter": "dummy_r", "region": regionVisit},
281 {"instrument": "DummyCam", "id": 1, "name": "one",
282 "physical_filter": "dummy_i"})
283 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()})
284 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract})
285 registry.insertDimensionData("patch",
286 {"skymap": "DummySkyMap", "tract": 0, "patch": 0,
287 "cell_x": 0, "cell_y": 0, "region": regionPatch})
288 registry.insertDimensionData("visit_detector_region",
289 {"instrument": "DummyCam", "visit": 0, "detector": 2,
290 "region": regionVisitDetector})
292 def getRegion(dataId):
293 return registry.expandDataId(dataId).region
295 # Get region for a tract
296 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0}))
297 # Attempt to get region for a non-existent tract
298 with self.assertRaises(LookupError):
299 getRegion({"skymap": "DummySkyMap", "tract": 1})
300 # Get region for a (tract, patch) combination
301 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0}))
302 # Get region for a non-existent (tract, patch) combination
303 with self.assertRaises(LookupError):
304 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1})
305 # Get region for a visit
306 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0}))
307 # Attempt to get region for a non-existent visit
308 with self.assertRaises(LookupError):
309 getRegion({"instrument": "DummyCam", "visit": 10})
310 # Get region for a (visit, detector) combination
311 self.assertEqual(regionVisitDetector,
312 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2}))
313 # Attempt to get region for a non-existent (visit, detector)
314 # combination. This returns None rather than raising because we don't
315 # want to require the region record to be present.
316 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3}))
317 # getRegion for a dataId containing no spatial dimensions should
318 # return None
319 self.assertIsNone(getRegion({"instrument": "DummyCam"}))
320 # Expanding a data ID with a mix of spatial dimensions should not fail,
321 # but it may not be implemented, so we don't test that we can get the
322 # region.
323 registry.expandDataId({"instrument": "DummyCam", "visit": 0, "detector": 2,
324 "skymap": "DummySkyMap", "tract": 0})
325 # Check if we can get the region for a skypix
326 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon)
327 # patch_htm7_overlap should not be empty
328 self.assertNotEqual(len(list(registry.queryDataIds(["patch", "htm7"]))), 0)
331class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
332 """Tests for `Registry` backed by a SQLite in-memory database.
334 This test case uses NameKeyCollectionManager.
335 """
336 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
339class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
340 """Tests for `Registry` backed by a SQLite in-memory database.
342 This test case uses SynthIntKeyCollectionManager.
343 """
344 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
347if __name__ == "__main__": 347 ↛ 348line 347 didn't jump to line 348, because the condition on line 347 was never true
348 unittest.main()