Coverage for tests/test_sqlite.py: 53%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

149 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import os 

24import os.path 

25import tempfile 

26import stat 

27import unittest 

28 

29import sqlalchemy 

30 

31from lsst.daf.butler import ddl 

32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

33from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

34from lsst.daf.butler.registry import Registry 

35from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

36 

37TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

38 

39 

40@contextmanager 

41def removeWritePermission(filename): 

42 mode = os.stat(filename).st_mode 

43 try: 

44 os.chmod(filename, stat.S_IREAD) 

45 yield 

46 finally: 

47 os.chmod(filename, mode) 

48 

49 

50def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

51 """Check whether we really can modify a database. 

52 

53 This intentionally allows any exception to be raised (not just 

54 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

55 but the Database was initialized (incorrectly) with writeable=True. 

56 """ 

57 try: 

58 with database.declareStaticTables(create=True) as context: 

59 table = context.addTable( 

60 "a", 

61 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

62 ) 

63 # Drop created table so that schema remains empty. 

64 database._metadata.drop_all(database._engine, tables=[table]) 

65 return True 

66 except Exception: 

67 return False 

68 

69 

70class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

71 """Tests for `SqliteDatabase` using a standard file-based database. 

72 """ 

73 

74 def setUp(self): 

75 self.root = makeTestTempDir(TESTDIR) 

76 

77 def tearDown(self): 

78 removeTestTempDir(self.root) 

79 

80 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

81 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

82 engine = SqliteDatabase.makeEngine(filename=filename) 

83 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

84 

85 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

86 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

87 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

88 

89 @contextmanager 

90 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

91 with removeWritePermission(database.filename): 

92 yield self.getNewConnection(database, writeable=False) 

93 

94 def testConnection(self): 

95 """Test that different ways of connecting to a SQLite database 

96 are equivalent. 

97 """ 

98 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

99 # Create a read-write database by passing in the filename. 

100 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

101 self.assertEqual(rwFromFilename.filename, filename) 

102 self.assertEqual(rwFromFilename.origin, 0) 

103 self.assertTrue(rwFromFilename.isWriteable()) 

104 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

105 # Create a read-write database via a URI. 

106 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

107 self.assertEqual(rwFromUri.filename, filename) 

108 self.assertEqual(rwFromUri.origin, 0) 

109 self.assertTrue(rwFromUri.isWriteable()) 

110 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

111 # We don't support SQLite URIs inside SQLAlchemy URIs. 

112 with self.assertRaises(NotImplementedError): 

113 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

114 

115 # Test read-only connections against a read-only file. 

116 with removeWritePermission(filename): 

117 # Create a read-only database by passing in the filename. 

118 roFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), 

119 origin=0, writeable=False) 

120 self.assertEqual(roFromFilename.filename, filename) 

121 self.assertEqual(roFromFilename.origin, 0) 

122 self.assertFalse(roFromFilename.isWriteable()) 

123 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

124 # Create a read-write database via a URI. 

125 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

126 self.assertEqual(roFromUri.filename, filename) 

127 self.assertEqual(roFromUri.origin, 0) 

128 self.assertFalse(roFromUri.isWriteable()) 

129 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

130 

131 def testTransactionLocking(self): 

132 # This (inherited) test can't run on SQLite because of our use of an 

133 # aggressive locking strategy there. 

134 pass 

135 

136 

137class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

138 """Tests for `SqliteDatabase` using an in-memory database. 

139 """ 

140 

141 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

142 engine = SqliteDatabase.makeEngine(filename=None) 

143 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

144 

145 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

146 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, 

147 writeable=writeable) 

148 

149 @contextmanager 

150 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

151 yield self.getNewConnection(database, writeable=False) 

152 

153 def testConnection(self): 

154 """Test that different ways of connecting to a SQLite database 

155 are equivalent. 

156 """ 

157 # Create an in-memory database by passing filename=None. 

158 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

159 self.assertIsNone(memFromFilename.filename) 

160 self.assertEqual(memFromFilename.origin, 0) 

161 self.assertTrue(memFromFilename.isWriteable()) 

162 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

163 # Create an in-memory database via a URI. 

164 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

165 self.assertIsNone(memFromUri.filename) 

166 self.assertEqual(memFromUri.origin, 0) 

167 self.assertTrue(memFromUri.isWriteable()) 

168 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

169 # We don't support SQLite URIs inside SQLAlchemy URIs. 

170 with self.assertRaises(NotImplementedError): 

171 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

172 # We don't support read-only in-memory databases. 

173 with self.assertRaises(NotImplementedError): 

174 SqliteDatabase.makeEngine(filename=None, writeable=False) 

175 

176 def testTransactionLocking(self): 

177 # This (inherited) test can't run on SQLite because of our use of an 

178 # aggressive locking strategy there. 

179 pass 

180 

181 

182class SqliteFileRegistryTests(RegistryTests): 

183 """Tests for `Registry` backed by a SQLite file-based database. 

184 

185 Note 

186 ---- 

187 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

188 defines methods that override `unittest.TestCase` methods. To make this 

189 work sublasses have to have this class first in the bases list. 

190 """ 

191 

192 def setUp(self): 

193 self.root = makeTestTempDir(TESTDIR) 

194 

195 def tearDown(self): 

196 removeTestTempDir(self.root) 

197 

198 @classmethod 

199 def getDataDir(cls) -> str: 

200 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

201 

202 def makeRegistry(self) -> Registry: 

203 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

204 config = self.makeRegistryConfig() 

205 config["db"] = f"sqlite:///{filename}" 

206 return Registry.createFromConfig(config, butlerRoot=self.root) 

207 

208 

209class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

210 """Tests for `Registry` backed by a SQLite file-based database. 

211 

212 This test case uses NameKeyCollectionManager and 

213 ByDimensionsDatasetRecordStorageManager. 

214 """ 

215 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

216 datasetsManager = \ 

217 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

218 

219 

220class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

221 """Tests for `Registry` backed by a SQLite file-based database. 

222 

223 This test case uses SynthIntKeyCollectionManager and 

224 ByDimensionsDatasetRecordStorageManager. 

225 """ 

226 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

227 datasetsManager = \ 

228 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

229 

230 

231class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

232 """Tests for `Registry` backed by a SQLite file-based database. 

233 

234 This test case uses NameKeyCollectionManager and 

235 ByDimensionsDatasetRecordStorageManagerUUID. 

236 """ 

237 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

238 datasetsManager = \ 

239 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

240 

241 

242class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

243 """Tests for `Registry` backed by a SQLite file-based database. 

244 

245 This test case uses SynthIntKeyCollectionManager and 

246 ByDimensionsDatasetRecordStorageManagerUUID. 

247 """ 

248 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

249 datasetsManager = \ 

250 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

251 

252 

253class SqliteMemoryRegistryTests(RegistryTests): 

254 """Tests for `Registry` backed by a SQLite in-memory database. 

255 """ 

256 

257 @classmethod 

258 def getDataDir(cls) -> str: 

259 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

260 

261 def makeRegistry(self) -> Registry: 

262 config = self.makeRegistryConfig() 

263 config["db"] = "sqlite://" 

264 return Registry.createFromConfig(config) 

265 

266 def testMissingAttributes(self): 

267 """Test for instantiating a registry against outdated schema which 

268 misses butler_attributes table. 

269 """ 

270 # TODO: Once we have stable gen3 schema everywhere this test can be 

271 # dropped (DM-27373). 

272 config = self.makeRegistryConfig() 

273 config["db"] = "sqlite://" 

274 with self.assertRaises(sqlalchemy.exc.OperationalError): 

275 Registry.fromConfig(config) 

276 

277 

278class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

279 """Tests for `Registry` backed by a SQLite in-memory database. 

280 

281 This test case uses NameKeyCollectionManager and 

282 ByDimensionsDatasetRecordStorageManager. 

283 """ 

284 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

285 datasetsManager = \ 

286 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

287 

288 

289class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

290 """Tests for `Registry` backed by a SQLite in-memory database. 

291 

292 This test case uses SynthIntKeyCollectionManager and 

293 ByDimensionsDatasetRecordStorageManager. 

294 """ 

295 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

296 datasetsManager = \ 

297 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

298 

299 

300class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

301 """Tests for `Registry` backed by a SQLite in-memory database. 

302 

303 This test case uses NameKeyCollectionManager and 

304 ByDimensionsDatasetRecordStorageManagerUUID. 

305 """ 

306 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

307 datasetsManager = \ 

308 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

309 

310 

311class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

312 """Tests for `Registry` backed by a SQLite in-memory database. 

313 

314 This test case uses SynthIntKeyCollectionManager and 

315 ByDimensionsDatasetRecordStorageManagerUUID. 

316 """ 

317 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

318 datasetsManager = \ 

319 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

320 

321 

322if __name__ == "__main__": 322 ↛ 323line 322 didn't jump to line 323, because the condition on line 322 was never true

323 unittest.main()