Coverage for tests/test_sqlite.py: 50%

156 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-24 02:27 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import os.path 

24import stat 

25import tempfile 

26import unittest 

27from contextlib import contextmanager 

28from typing import Optional 

29 

30import sqlalchemy 

31from lsst.daf.butler import ddl 

32from lsst.daf.butler.registry import Registry 

33from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

34from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

35from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

36 

37TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

38 

39 

40@contextmanager 

41def removeWritePermission(filename): 

42 mode = os.stat(filename).st_mode 

43 try: 

44 os.chmod(filename, stat.S_IREAD) 

45 yield 

46 finally: 

47 os.chmod(filename, mode) 

48 

49 

50def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

51 """Check whether we really can modify a database. 

52 

53 This intentionally allows any exception to be raised (not just 

54 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

55 but the Database was initialized (incorrectly) with writeable=True. 

56 """ 

57 try: 

58 with database.declareStaticTables(create=True) as context: 

59 table = context.addTable( 

60 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

61 ) 

62 # Drop created table so that schema remains empty. 

63 database._metadata.drop_all(database._engine, tables=[table]) 

64 return True 

65 except Exception: 

66 return False 

67 

68 

69class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

70 """Tests for `SqliteDatabase` using a standard file-based database.""" 

71 

72 def setUp(self): 

73 self.root = makeTestTempDir(TESTDIR) 

74 

75 def tearDown(self): 

76 removeTestTempDir(self.root) 

77 

78 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

79 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

80 engine = SqliteDatabase.makeEngine(filename=filename) 

81 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

82 

83 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

84 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

85 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

86 

87 @contextmanager 

88 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

89 with removeWritePermission(database.filename): 

90 yield self.getNewConnection(database, writeable=False) 

91 

92 def testConnection(self): 

93 """Test that different ways of connecting to a SQLite database 

94 are equivalent. 

95 """ 

96 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

97 # Create a read-write database by passing in the filename. 

98 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

99 self.assertEqual(rwFromFilename.filename, filename) 

100 self.assertEqual(rwFromFilename.origin, 0) 

101 self.assertTrue(rwFromFilename.isWriteable()) 

102 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

103 # Create a read-write database via a URI. 

104 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

105 self.assertEqual(rwFromUri.filename, filename) 

106 self.assertEqual(rwFromUri.origin, 0) 

107 self.assertTrue(rwFromUri.isWriteable()) 

108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

109 # We don't support SQLite URIs inside SQLAlchemy URIs. 

110 with self.assertRaises(NotImplementedError): 

111 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

112 

113 # Test read-only connections against a read-only file. 

114 with removeWritePermission(filename): 

115 # Create a read-only database by passing in the filename. 

116 roFromFilename = SqliteDatabase.fromEngine( 

117 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False 

118 ) 

119 self.assertEqual(roFromFilename.filename, filename) 

120 self.assertEqual(roFromFilename.origin, 0) 

121 self.assertFalse(roFromFilename.isWriteable()) 

122 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

123 # Create a read-write database via a URI. 

124 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

125 self.assertEqual(roFromUri.filename, filename) 

126 self.assertEqual(roFromUri.origin, 0) 

127 self.assertFalse(roFromUri.isWriteable()) 

128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

129 

130 def testTransactionLocking(self): 

131 # This (inherited) test can't run on SQLite because of our use of an 

132 # aggressive locking strategy there. 

133 pass 

134 

135 

136class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

137 """Tests for `SqliteDatabase` using an in-memory database.""" 

138 

139 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

140 engine = SqliteDatabase.makeEngine(filename=None) 

141 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

142 

143 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

144 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable) 

145 

146 @contextmanager 

147 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

148 yield self.getNewConnection(database, writeable=False) 

149 

150 def testConnection(self): 

151 """Test that different ways of connecting to a SQLite database 

152 are equivalent. 

153 """ 

154 # Create an in-memory database by passing filename=None. 

155 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

156 self.assertIsNone(memFromFilename.filename) 

157 self.assertEqual(memFromFilename.origin, 0) 

158 self.assertTrue(memFromFilename.isWriteable()) 

159 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

160 # Create an in-memory database via a URI. 

161 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

162 self.assertIsNone(memFromUri.filename) 

163 self.assertEqual(memFromUri.origin, 0) 

164 self.assertTrue(memFromUri.isWriteable()) 

165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

166 # We don't support SQLite URIs inside SQLAlchemy URIs. 

167 with self.assertRaises(NotImplementedError): 

168 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

169 # We don't support read-only in-memory databases. 

170 with self.assertRaises(NotImplementedError): 

171 SqliteDatabase.makeEngine(filename=None, writeable=False) 

172 

173 def testTransactionLocking(self): 

174 # This (inherited) test can't run on SQLite because of our use of an 

175 # aggressive locking strategy there. 

176 pass 

177 

178 

179class SqliteFileRegistryTests(RegistryTests): 

180 """Tests for `Registry` backed by a SQLite file-based database. 

181 

182 Note 

183 ---- 

184 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

185 defines methods that override `unittest.TestCase` methods. To make this 

186 work sublasses have to have this class first in the bases list. 

187 """ 

188 

189 def setUp(self): 

190 self.root = makeTestTempDir(TESTDIR) 

191 

192 def tearDown(self): 

193 removeTestTempDir(self.root) 

194 

195 @classmethod 

196 def getDataDir(cls) -> str: 

197 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

198 

199 def makeRegistry(self, share_repo_with: Optional[Registry] = None) -> Registry: 

200 if share_repo_with is None: 

201 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

202 else: 

203 filename = share_repo_with._db.filename 

204 config = self.makeRegistryConfig() 

205 config["db"] = f"sqlite:///{filename}" 

206 if share_repo_with is None: 

207 return Registry.createFromConfig(config, butlerRoot=self.root) 

208 else: 

209 return Registry.fromConfig(config, butlerRoot=self.root) 

210 

211 

212class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

213 """Tests for `Registry` backed by a SQLite file-based database. 

214 

215 This test case uses NameKeyCollectionManager and 

216 ByDimensionsDatasetRecordStorageManager. 

217 """ 

218 

219 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

220 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

221 

222 

223class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

224 """Tests for `Registry` backed by a SQLite file-based database. 

225 

226 This test case uses SynthIntKeyCollectionManager and 

227 ByDimensionsDatasetRecordStorageManager. 

228 """ 

229 

230 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

231 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

232 

233 

234class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

235 """Tests for `Registry` backed by a SQLite file-based database. 

236 

237 This test case uses NameKeyCollectionManager and 

238 ByDimensionsDatasetRecordStorageManagerUUID. 

239 """ 

240 

241 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

242 datasetsManager = ( 

243 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

244 ) 

245 

246 

247class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

248 """Tests for `Registry` backed by a SQLite file-based database. 

249 

250 This test case uses SynthIntKeyCollectionManager and 

251 ByDimensionsDatasetRecordStorageManagerUUID. 

252 """ 

253 

254 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

255 datasetsManager = ( 

256 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

257 ) 

258 

259 

260class SqliteMemoryRegistryTests(RegistryTests): 

261 """Tests for `Registry` backed by a SQLite in-memory database.""" 

262 

263 @classmethod 

264 def getDataDir(cls) -> str: 

265 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

266 

267 def makeRegistry(self, share_repo_with: Optional[Registry] = None) -> Optional[Registry]: 

268 if share_repo_with is not None: 

269 return None 

270 config = self.makeRegistryConfig() 

271 config["db"] = "sqlite://" 

272 return Registry.createFromConfig(config) 

273 

274 def testMissingAttributes(self): 

275 """Test for instantiating a registry against outdated schema which 

276 misses butler_attributes table. 

277 """ 

278 # TODO: Once we have stable gen3 schema everywhere this test can be 

279 # dropped (DM-27373). 

280 config = self.makeRegistryConfig() 

281 config["db"] = "sqlite://" 

282 with self.assertRaises(sqlalchemy.exc.OperationalError): 

283 Registry.fromConfig(config) 

284 

285 

286class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

287 """Tests for `Registry` backed by a SQLite in-memory database. 

288 

289 This test case uses NameKeyCollectionManager and 

290 ByDimensionsDatasetRecordStorageManager. 

291 """ 

292 

293 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

294 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

295 

296 

297class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

298 """Tests for `Registry` backed by a SQLite in-memory database. 

299 

300 This test case uses SynthIntKeyCollectionManager and 

301 ByDimensionsDatasetRecordStorageManager. 

302 """ 

303 

304 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

305 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

306 

307 

308class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

309 """Tests for `Registry` backed by a SQLite in-memory database. 

310 

311 This test case uses NameKeyCollectionManager and 

312 ByDimensionsDatasetRecordStorageManagerUUID. 

313 """ 

314 

315 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

316 datasetsManager = ( 

317 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

318 ) 

319 

320 

321class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

322 """Tests for `Registry` backed by a SQLite in-memory database. 

323 

324 This test case uses SynthIntKeyCollectionManager and 

325 ByDimensionsDatasetRecordStorageManagerUUID. 

326 """ 

327 

328 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

329 datasetsManager = ( 

330 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

331 ) 

332 

333 

334if __name__ == "__main__": 334 ↛ 335line 334 didn't jump to line 335, because the condition on line 334 was never true

335 unittest.main()