Coverage for tests/test_sqlite.py : 26%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from contextlib import contextmanager
23import itertools
24import os
25import os.path
26import shutil
27import stat
28import tempfile
29import unittest
31import sqlalchemy
33from lsst.sphgeom import ConvexPolygon, UnitVector3d
35from lsst.daf.butler import ddl
36from lsst.daf.butler.registry import RegistryConfig
37from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase
38from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
39from lsst.daf.butler.registry import Registry
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44@contextmanager
45def removeWritePermission(filename):
46 mode = os.stat(filename).st_mode
47 try:
48 os.chmod(filename, stat.S_IREAD)
49 yield
50 finally:
51 os.chmod(filename, mode)
54def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool:
55 """Check whether we really can modify a database.
57 This intentionally allows any exception to be raised (not just
58 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only
59 but the Database was initialized (incorrectly) with writeable=True.
60 """
61 try:
62 with database.declareStaticTables(create=True) as context:
63 context.addTable(
64 "a",
65 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)])
66 )
67 return True
68 except Exception:
69 return False
72class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests):
73 """Tests for `SqliteDatabase` using a standard file-based database.
74 """
76 def setUp(self):
77 self.root = tempfile.mkdtemp(dir=TESTDIR)
79 def tearDown(self):
80 if self.root is not None and os.path.exists(self.root):
81 shutil.rmtree(self.root, ignore_errors=True)
83 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
84 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
85 connection = SqliteDatabase.connect(filename=filename)
86 return SqliteDatabase.fromConnection(connection=connection, origin=origin)
88 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
89 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable)
90 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection,
91 writeable=writeable)
93 @contextmanager
94 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
95 with removeWritePermission(database.filename):
96 yield self.getNewConnection(database, writeable=False)
98 def testConnection(self):
99 """Test that different ways of connecting to a SQLite database
100 are equivalent.
101 """
102 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
103 # Create a read-write database by passing in the filename.
104 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0)
105 self.assertEqual(rwFromFilename.filename, filename)
106 self.assertEqual(rwFromFilename.origin, 0)
107 self.assertTrue(rwFromFilename.isWriteable())
108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename))
109 # Create a read-write database via a URI.
110 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0)
111 self.assertEqual(rwFromUri.filename, filename)
112 self.assertEqual(rwFromUri.origin, 0)
113 self.assertTrue(rwFromUri.isWriteable())
114 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri))
115 # We don't support SQLite URIs inside SQLAlchemy URIs.
116 with self.assertRaises(NotImplementedError):
117 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true")
119 # Test read-only connections against a read-only file.
120 with removeWritePermission(filename):
121 # Create a read-only database by passing in the filename.
122 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename),
123 origin=0, writeable=False)
124 self.assertEqual(roFromFilename.filename, filename)
125 self.assertEqual(roFromFilename.origin, 0)
126 self.assertFalse(roFromFilename.isWriteable())
127 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename))
128 # Create a read-write database via a URI.
129 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False)
130 self.assertEqual(roFromUri.filename, filename)
131 self.assertEqual(roFromUri.origin, 0)
132 self.assertFalse(roFromUri.isWriteable())
133 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri))
136class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests):
137 """Tests for `SqliteDatabase` using an in-memory database.
138 """
140 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase:
141 connection = SqliteDatabase.connect(filename=None)
142 return SqliteDatabase.fromConnection(connection=connection, origin=origin)
144 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase:
145 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection,
146 writeable=writeable)
148 @contextmanager
149 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase:
150 yield self.getNewConnection(database, writeable=False)
152 def testConnection(self):
153 """Test that different ways of connecting to a SQLite database
154 are equivalent.
155 """
156 # Create an in-memory database by passing filename=None.
157 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0)
158 self.assertIsNone(memFromFilename.filename)
159 self.assertEqual(memFromFilename.origin, 0)
160 self.assertTrue(memFromFilename.isWriteable())
161 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename))
162 # Create an in-memory database via a URI.
163 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0)
164 self.assertIsNone(memFromUri.filename)
165 self.assertEqual(memFromUri.origin, 0)
166 self.assertTrue(memFromUri.isWriteable())
167 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri))
168 # We don't support SQLite URIs inside SQLAlchemy URIs.
169 with self.assertRaises(NotImplementedError):
170 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true")
171 # We don't support read-only in-memory databases.
172 with self.assertRaises(NotImplementedError):
173 SqliteDatabase.connect(filename=None, writeable=False)
176class SqliteFileRegistryTestCase(unittest.TestCase, RegistryTests):
177 """Tests for `Registry` backed by a SQLite file-based database.
178 """
180 def setUp(self):
181 self.root = tempfile.mkdtemp(dir=TESTDIR)
183 def tearDown(self):
184 if self.root is not None and os.path.exists(self.root):
185 shutil.rmtree(self.root, ignore_errors=True)
187 def makeRegistry(self) -> Registry:
188 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
189 config = RegistryConfig()
190 config["db"] = f"sqlite:///{filename}"
191 return Registry.fromConfig(config, create=True, butlerRoot=self.root)
194class SqliteMemoryRegistryTestCase(unittest.TestCase, RegistryTests):
195 """Tests for `Registry` backed by a SQLite in-memory database.
196 """
198 def makeRegistry(self) -> Registry:
199 config = RegistryConfig()
200 config["db"] = f"sqlite://"
201 return Registry.fromConfig(config, create=True)
203 def testRegions(self):
204 """Tests for using region fields in `Registry` dimensions.
205 """
206 # TODO: the test regions used here are enormous (significant fractions
207 # of the sphere), and that makes this test prohibitively slow on
208 # most real databases. These should be made more realistic, and the
209 # test moved to daf/butler/registry/tests/registry.py.
210 registry = self.makeRegistry()
211 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0),
212 UnitVector3d(0, 1, 0),
213 UnitVector3d(0, 0, 1)))
214 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0),
215 UnitVector3d(0, 1, 0),
216 UnitVector3d(0, 0, 1)))
217 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0),
218 UnitVector3d(0, 1, 1),
219 UnitVector3d(0, 0, 1)))
220 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0),
221 UnitVector3d(0, 1, 0),
222 UnitVector3d(0, 1, 1)))
223 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2):
224 self.assertNotEqual(a, b)
226 # This depends on current dimensions.yaml definitions
227 self.assertEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0)
229 # Add some dimension entries
230 registry.insertDimensionData("instrument", {"name": "DummyCam"})
231 registry.insertDimensionData("physical_filter",
232 {"instrument": "DummyCam", "name": "dummy_r", "abstract_filter": "r"},
233 {"instrument": "DummyCam", "name": "dummy_i", "abstract_filter": "i"})
234 for detector in (1, 2, 3, 4, 5):
235 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector,
236 "full_name": str(detector)})
237 registry.insertDimensionData("visit",
238 {"instrument": "DummyCam", "id": 0, "name": "zero",
239 "physical_filter": "dummy_r", "region": regionVisit},
240 {"instrument": "DummyCam", "id": 1, "name": "one",
241 "physical_filter": "dummy_i"})
242 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()})
243 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract})
244 registry.insertDimensionData("patch",
245 {"skymap": "DummySkyMap", "tract": 0, "patch": 0,
246 "cell_x": 0, "cell_y": 0, "region": regionPatch})
247 registry.insertDimensionData("visit_detector_region",
248 {"instrument": "DummyCam", "visit": 0, "detector": 2,
249 "region": regionVisitDetector})
251 def getRegion(dataId):
252 return registry.expandDataId(dataId).region
254 # Get region for a tract
255 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0}))
256 # Attempt to get region for a non-existent tract
257 with self.assertRaises(LookupError):
258 getRegion({"skymap": "DummySkyMap", "tract": 1})
259 # Get region for a (tract, patch) combination
260 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0}))
261 # Get region for a non-existent (tract, patch) combination
262 with self.assertRaises(LookupError):
263 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1})
264 # Get region for a visit
265 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0}))
266 # Attempt to get region for a non-existent visit
267 with self.assertRaises(LookupError):
268 getRegion({"instrument": "DummyCam", "visit": 10})
269 # Get region for a (visit, detector) combination
270 self.assertEqual(regionVisitDetector,
271 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2}))
272 # Attempt to get region for a non-existent (visit, detector)
273 # combination. This returns None rather than raising because we don't
274 # want to require the region record to be present.
275 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3}))
276 # getRegion for a dataId containing no spatial dimensions should
277 # return None
278 self.assertIsNone(getRegion({"instrument": "DummyCam"}))
279 # getRegion for a mix of spatial dimensions should return
280 # NotImplemented, at least until we get it implemented.
281 self.assertIs(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2,
282 "skymap": "DummySkyMap", "tract": 0}),
283 NotImplemented)
284 # Check if we can get the region for a skypix
285 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon)
286 # patch_htm7_overlap should not be empty
287 self.assertNotEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0)
290if __name__ == "__main__": 290 ↛ 291line 290 didn't jump to line 291, because the condition on line 290 was never true
291 unittest.main()