Coverage for tests/test_sqlite.py: 40%

149 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-04 02:55 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import os 

29import os.path 

30import stat 

31import tempfile 

32import unittest 

33from contextlib import contextmanager 

34 

35import sqlalchemy 

36from lsst.daf.butler import ddl 

37from lsst.daf.butler.registry import _RegistryFactory 

38from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

39from lsst.daf.butler.registry.sql_registry import SqlRegistry 

40from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

41from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

42 

43TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

44 

45 

46@contextmanager 

47def removeWritePermission(filename): 

48 """Remove the write permission on a file.""" 

49 mode = os.stat(filename).st_mode 

50 try: 

51 os.chmod(filename, stat.S_IREAD) 

52 yield 

53 finally: 

54 os.chmod(filename, mode) 

55 

56 

57def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

58 """Check whether we really can modify a database. 

59 

60 This intentionally allows any exception to be raised (not just 

61 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

62 but the Database was initialized (incorrectly) with writeable=True. 

63 """ 

64 try: 

65 with database.declareStaticTables(create=True) as context: 

66 table = context.addTable( 

67 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

68 ) 

69 # Drop created table so that schema remains empty. 

70 database._metadata.drop_all(database._engine, tables=[table]) 

71 return True 

72 except Exception: 

73 return False 

74 

75 

76class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

77 """Tests for `SqliteDatabase` using a standard file-based database.""" 

78 

79 def setUp(self): 

80 self.root = makeTestTempDir(TESTDIR) 

81 

82 def tearDown(self): 

83 removeTestTempDir(self.root) 

84 

85 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

86 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

87 engine = SqliteDatabase.makeEngine(filename=filename) 

88 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

89 

90 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

91 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

92 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

93 

94 @contextmanager 

95 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

96 with removeWritePermission(database.filename): 

97 yield self.getNewConnection(database, writeable=False) 

98 

99 def testConnection(self): 

100 """Test that different ways of connecting to a SQLite database 

101 are equivalent. 

102 """ 

103 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

104 # Create a read-write database by passing in the filename. 

105 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

106 self.assertEqual(rwFromFilename.filename, filename) 

107 self.assertEqual(rwFromFilename.origin, 0) 

108 self.assertTrue(rwFromFilename.isWriteable()) 

109 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

110 # Create a read-write database via a URI. 

111 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

112 self.assertEqual(rwFromUri.filename, filename) 

113 self.assertEqual(rwFromUri.origin, 0) 

114 self.assertTrue(rwFromUri.isWriteable()) 

115 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

116 # We don't support SQLite URIs inside SQLAlchemy URIs. 

117 with self.assertRaises(NotImplementedError): 

118 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

119 

120 # Test read-only connections against a read-only file. 

121 with removeWritePermission(filename): 

122 # Create a read-only database by passing in the filename. 

123 roFromFilename = SqliteDatabase.fromEngine( 

124 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False 

125 ) 

126 self.assertEqual(roFromFilename.filename, filename) 

127 self.assertEqual(roFromFilename.origin, 0) 

128 self.assertFalse(roFromFilename.isWriteable()) 

129 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

130 # Create a read-write database via a URI. 

131 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

132 self.assertEqual(roFromUri.filename, filename) 

133 self.assertEqual(roFromUri.origin, 0) 

134 self.assertFalse(roFromUri.isWriteable()) 

135 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

136 

137 def testTransactionLocking(self): 

138 # This (inherited) test can't run on SQLite because of our use of an 

139 # aggressive locking strategy there. 

140 pass 

141 

142 

143class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

144 """Tests for `SqliteDatabase` using an in-memory database.""" 

145 

146 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

147 engine = SqliteDatabase.makeEngine(filename=None) 

148 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

149 

150 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

151 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable) 

152 

153 @contextmanager 

154 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

155 yield self.getNewConnection(database, writeable=False) 

156 

157 def testConnection(self): 

158 """Test that different ways of connecting to a SQLite database 

159 are equivalent. 

160 """ 

161 # Create an in-memory database by passing filename=None. 

162 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

163 self.assertIsNone(memFromFilename.filename) 

164 self.assertEqual(memFromFilename.origin, 0) 

165 self.assertTrue(memFromFilename.isWriteable()) 

166 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

167 # Create an in-memory database via a URI. 

168 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

169 self.assertIsNone(memFromUri.filename) 

170 self.assertEqual(memFromUri.origin, 0) 

171 self.assertTrue(memFromUri.isWriteable()) 

172 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

173 # We don't support SQLite URIs inside SQLAlchemy URIs. 

174 with self.assertRaises(NotImplementedError): 

175 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

176 # We don't support read-only in-memory databases. 

177 with self.assertRaises(NotImplementedError): 

178 SqliteDatabase.makeEngine(filename=None, writeable=False) 

179 

180 def testTransactionLocking(self): 

181 # This (inherited) test can't run on SQLite because of our use of an 

182 # aggressive locking strategy there. 

183 pass 

184 

185 

186class SqliteFileRegistryTests(RegistryTests): 

187 """Tests for `Registry` backed by a SQLite file-based database. 

188 

189 Notes 

190 ----- 

191 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

192 defines methods that override `unittest.TestCase` methods. To make this 

193 work sublasses have to have this class first in the bases list. 

194 """ 

195 

196 def setUp(self): 

197 self.root = makeTestTempDir(TESTDIR) 

198 

199 def tearDown(self): 

200 removeTestTempDir(self.root) 

201 

202 @classmethod 

203 def getDataDir(cls) -> str: 

204 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

205 

206 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry: 

207 if share_repo_with is None: 

208 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

209 else: 

210 filename = share_repo_with._db.filename 

211 config = self.makeRegistryConfig() 

212 config["db"] = f"sqlite:///{filename}" 

213 if share_repo_with is None: 

214 return _RegistryFactory(config).create_from_config(butlerRoot=self.root) 

215 else: 

216 return _RegistryFactory(config).from_config(butlerRoot=self.root) 

217 

218 

219class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

220 """Tests for `Registry` backed by a SQLite file-based database. 

221 

222 This test case uses NameKeyCollectionManager and 

223 ByDimensionsDatasetRecordStorageManagerUUID. 

224 """ 

225 

226 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

227 datasetsManager = ( 

228 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

229 ) 

230 

231 

232class ClonedSqliteFileRegistryNameKeyCollMgrUUIDTestCase( 

233 SqliteFileRegistryNameKeyCollMgrUUIDTestCase, unittest.TestCase 

234): 

235 """Test that NameKeyCollectionManager still works after cloning.""" 

236 

237 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry: 

238 original = super().makeRegistry(share_repo_with) 

239 return original.copy() 

240 

241 

242class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

243 """Tests for `Registry` backed by a SQLite file-based database. 

244 

245 This test case uses SynthIntKeyCollectionManager and 

246 ByDimensionsDatasetRecordStorageManagerUUID. 

247 """ 

248 

249 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

250 datasetsManager = ( 

251 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

252 ) 

253 

254 

255class SqliteMemoryRegistryTests(RegistryTests): 

256 """Tests for `Registry` backed by a SQLite in-memory database.""" 

257 

258 @classmethod 

259 def getDataDir(cls) -> str: 

260 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

261 

262 def makeRegistry(self, share_repo_with: SqlRegistry | None = None) -> SqlRegistry | None: 

263 if share_repo_with is not None: 

264 return None 

265 config = self.makeRegistryConfig() 

266 config["db"] = "sqlite://" 

267 return _RegistryFactory(config).create_from_config() 

268 

269 def testMissingAttributes(self): 

270 """Test for instantiating a registry against outdated schema which 

271 misses butler_attributes table. 

272 """ 

273 # TODO: Once we have stable gen3 schema everywhere this test can be 

274 # dropped (DM-27373). 

275 config = self.makeRegistryConfig() 

276 config["db"] = "sqlite://" 

277 with self.assertRaises(LookupError): 

278 _RegistryFactory(config).from_config() 

279 

280 

281class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

282 """Tests for `Registry` backed by a SQLite in-memory database. 

283 

284 This test case uses NameKeyCollectionManager and 

285 ByDimensionsDatasetRecordStorageManagerUUID. 

286 """ 

287 

288 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

289 datasetsManager = ( 

290 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

291 ) 

292 

293 

294class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

295 """Tests for `Registry` backed by a SQLite in-memory database. 

296 

297 This test case uses SynthIntKeyCollectionManager and 

298 ByDimensionsDatasetRecordStorageManagerUUID. 

299 """ 

300 

301 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

302 datasetsManager = ( 

303 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

304 ) 

305 

306 

307class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

308 """Tests for `Registry` backed by a SQLite in-memory database. 

309 

310 This test case uses version schema with ingest_date as nanoseconds instead 

311 of DATETIME. This tests can be removed when/if we switch to nanoseconds as 

312 default. 

313 """ 

314 

315 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

316 datasetsManager = { 

317 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID", 

318 "schema_version": "2.0.0", 

319 } 

320 

321 

322if __name__ == "__main__": 

323 unittest.main()