Coverage for tests/test_sqlite.py : 30%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from contextlib import contextmanager
23import itertools
24import os
25import os.path
26import shutil
27import stat
28import tempfile
29import unittest
31import sqlalchemy
33from lsst.sphgeom import ConvexPolygon, UnitVector3d
35from lsst.daf.butler import ddl
36from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
37from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
38from lsst.daf.butler.registry import Registry
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
43@contextmanager
44def removeWritePermission(filename):
45 mode = os.stat(filename).st_mode
46 try:
47 os.chmod(filename, stat.S_IREAD)
48 yield
49 finally:
50 os.chmod(filename, mode)
53def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
54 """Check whether we really can modify a database.
56 This intentionally allows any exception to be raised (not just
57 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
58 but the Database was initialized (incorrectly) with writeable=True.
59 """
60 try:
61 with database.declareStaticTables(create=True) as context:
62 context.addTable(
63 "a",
64 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
65 )
66 return True
67 except Exception:
68 return False
71class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
72 """Tests for `SqliteDatabase` using a standard file-based database.
73 """
75 def setUp(self):
76 self.root = tempfile.mkdtemp(dir=TESTDIR)
78 def tearDown(self):
79 if self.root is not None and os.path.exists(self.root):
80 shutil.rmtree(self.root, ignore_errors=True)
82 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
83 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
84 connection = SqliteDatabase.connect(filename=filename)
85 return SqliteDatabase.fromConnection(connection=connection, origin=origin)
87 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
88 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable)
89 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection,
90 writeable=writeable)
92 @contextmanager
93 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
94 with removeWritePermission(database.filename):
95 yield self.getNewConnection(database, writeable=False)
97 def testConnection(self):
98 """Test that different ways of connecting to a SQLite database
99 are equivalent.
100 """
101 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
102 # Create a read-write database by passing in the filename.
103 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0)
104 self.assertEqual(rwFromFilename.filename, filename)
105 self.assertEqual(rwFromFilename.origin, 0)
106 self.assertTrue(rwFromFilename.isWriteable())
107 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
108 # Create a read-write database via a URI.
109 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
110 self.assertEqual(rwFromUri.filename, filename)
111 self.assertEqual(rwFromUri.origin, 0)
112 self.assertTrue(rwFromUri.isWriteable())
113 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
114 # We don't support SQLite URIs inside SQLAlchemy URIs.
115 with self.assertRaises(NotImplementedError):
116 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true")
118 # Test read-only connections against a read-only file.
119 with removeWritePermission(filename):
120 # Create a read-only database by passing in the filename.
121 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename),
122 origin=0, writeable=False)
123 self.assertEqual(roFromFilename.filename, filename)
124 self.assertEqual(roFromFilename.origin, 0)
125 self.assertFalse(roFromFilename.isWriteable())
126 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
127 # Create a read-write database via a URI.
128 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
129 self.assertEqual(roFromUri.filename, filename)
130 self.assertEqual(roFromUri.origin, 0)
131 self.assertFalse(roFromUri.isWriteable())
132 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
135class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
136 """Tests for `SqliteDatabase` using an in-memory database.
137 """
139 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
140 connection = SqliteDatabase.connect(filename=None)
141 return SqliteDatabase.fromConnection(connection=connection, origin=origin)
143 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
144 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection,
145 writeable=writeable)
147 @contextmanager
148 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
149 yield self.getNewConnection(database, writeable=False)
151 def testConnection(self):
152 """Test that different ways of connecting to a SQLite database
153 are equivalent.
154 """
155 # Create an in-memory database by passing filename=None.
156 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0)
157 self.assertIsNone(memFromFilename.filename)
158 self.assertEqual(memFromFilename.origin, 0)
159 self.assertTrue(memFromFilename.isWriteable())
160 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
161 # Create an in-memory database via a URI.
162 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
163 self.assertIsNone(memFromUri.filename)
164 self.assertEqual(memFromUri.origin, 0)
165 self.assertTrue(memFromUri.isWriteable())
166 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
167 # We don't support SQLite URIs inside SQLAlchemy URIs.
168 with self.assertRaises(NotImplementedError):
169 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true")
170 # We don't support read-only in-memory databases.
171 with self.assertRaises(NotImplementedError):
172 SqliteDatabase.connect(filename=None, writeable=False)
175class SqliteFileRegistryTests(RegistryTests):
176 """Tests for `Registry` backed by a SQLite file-based database.
178 Note
179 ----
180 This is not a subclass of `unittest.TestCase` but to avoid repetition it
181 defines methods that override `unittest.TestCase` methods. To make this
182 work sublasses have to have this class first in the bases list.
183 """
185 def setUp(self):
186 self.root = tempfile.mkdtemp(dir=TESTDIR)
188 def tearDown(self):
189 if self.root is not None and os.path.exists(self.root):
190 shutil.rmtree(self.root, ignore_errors=True)
192 @classmethod
193 def getDataDir(cls) -> str:
194 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
196 def makeRegistry(self) -> Registry:
197 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
198 config = self.makeRegistryConfig()
199 config["db"] = f"sqlite:///{filename}"
200 return Registry.fromConfig(config, create=True, butlerRoot=self.root)
203class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase):
204 """Tests for `Registry` backed by a SQLite file-based database.
206 This test case uses NameKeyCollectionManager.
207 """
208 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
211class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase):
212 """Tests for `Registry` backed by a SQLite file-based database.
214 This test case uses SynthIntKeyCollectionManager.
215 """
216 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
219class SqliteMemoryRegistryTests(RegistryTests):
220 """Tests for `Registry` backed by a SQLite in-memory database.
221 """
223 @classmethod
224 def getDataDir(cls) -> str:
225 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry"))
227 def makeRegistry(self) -> Registry:
228 config = self.makeRegistryConfig()
229 config["db"] = "sqlite://"
230 return Registry.fromConfig(config, create=True)
232 def testRegions(self):
233 """Tests for using region fields in `Registry` dimensions.
234 """
235 # TODO: the test regions used here are enormous (significant fractions
236 # of the sphere), and that makes this test prohibitively slow on
237 # most real databases. These should be made more realistic, and the
238 # test moved to daf/butler/registry/tests/registry.py.
239 registry = self.makeRegistry()
240 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0),
241 UnitVector3d(0, 1, 0),
242 UnitVector3d(0, 0, 1)))
243 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0),
244 UnitVector3d(0, 1, 0),
245 UnitVector3d(0, 0, 1)))
246 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0),
247 UnitVector3d(0, 1, 1),
248 UnitVector3d(0, 0, 1)))
249 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0),
250 UnitVector3d(0, 1, 0),
251 UnitVector3d(0, 1, 1)))
252 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2):
253 self.assertNotEqual(a, b)
255 # This depends on current dimensions.yaml definitions
256 self.assertEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0)
258 # Add some dimension entries
259 registry.insertDimensionData("instrument", {"name": "DummyCam"})
260 registry.insertDimensionData("physical_filter",
261 {"instrument": "DummyCam", "name": "dummy_r", "abstract_filter": "r"},
262 {"instrument": "DummyCam", "name": "dummy_i", "abstract_filter": "i"})
263 for detector in (1, 2, 3, 4, 5):
264 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector,
265 "full_name": str(detector)})
266 registry.insertDimensionData("visit",
267 {"instrument": "DummyCam", "id": 0, "name": "zero",
268 "physical_filter": "dummy_r", "region": regionVisit},
269 {"instrument": "DummyCam", "id": 1, "name": "one",
270 "physical_filter": "dummy_i"})
271 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()})
272 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract})
273 registry.insertDimensionData("patch",
274 {"skymap": "DummySkyMap", "tract": 0, "patch": 0,
275 "cell_x": 0, "cell_y": 0, "region": regionPatch})
276 registry.insertDimensionData("visit_detector_region",
277 {"instrument": "DummyCam", "visit": 0, "detector": 2,
278 "region": regionVisitDetector})
280 def getRegion(dataId):
281 return registry.expandDataId(dataId).region
283 # Get region for a tract
284 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0}))
285 # Attempt to get region for a non-existent tract
286 with self.assertRaises(LookupError):
287 getRegion({"skymap": "DummySkyMap", "tract": 1})
288 # Get region for a (tract, patch) combination
289 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0}))
290 # Get region for a non-existent (tract, patch) combination
291 with self.assertRaises(LookupError):
292 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1})
293 # Get region for a visit
294 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0}))
295 # Attempt to get region for a non-existent visit
296 with self.assertRaises(LookupError):
297 getRegion({"instrument": "DummyCam", "visit": 10})
298 # Get region for a (visit, detector) combination
299 self.assertEqual(regionVisitDetector,
300 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2}))
301 # Attempt to get region for a non-existent (visit, detector)
302 # combination. This returns None rather than raising because we don't
303 # want to require the region record to be present.
304 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3}))
305 # getRegion for a dataId containing no spatial dimensions should
306 # return None
307 self.assertIsNone(getRegion({"instrument": "DummyCam"}))
308 # Expanding a data ID with a mix of spatial dimensions should not fail,
309 # but it may not be implemented, so we don't test that we can get the
310 # region.
311 registry.expandDataId({"instrument": "DummyCam", "visit": 0, "detector": 2,
312 "skymap": "DummySkyMap", "tract": 0})
313 # Check if we can get the region for a skypix
314 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon)
315 # patch_htm7_overlap should not be empty
316 self.assertNotEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0)
319class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
320 """Tests for `Registry` backed by a SQLite in-memory database.
322 This test case uses NameKeyCollectionManager.
323 """
324 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager"
327class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests):
328 """Tests for `Registry` backed by a SQLite in-memory database.
330 This test case uses SynthIntKeyCollectionManager.
331 """
332 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager"
335if __name__ == "__main__": 335 ↛ 336line 335 didn't jump to line 336, because the condition on line 335 was never true
336 unittest.main()