Coverage for tests/test_sqlite.py: 53%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

149 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import os.path 

24import stat 

25import tempfile 

26import unittest 

27from contextlib import contextmanager 

28 

29import sqlalchemy 

30from lsst.daf.butler import ddl 

31from lsst.daf.butler.registry import Registry 

32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

33from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

34from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39@contextmanager 

40def removeWritePermission(filename): 

41 mode = os.stat(filename).st_mode 

42 try: 

43 os.chmod(filename, stat.S_IREAD) 

44 yield 

45 finally: 

46 os.chmod(filename, mode) 

47 

48 

49def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

50 """Check whether we really can modify a database. 

51 

52 This intentionally allows any exception to be raised (not just 

53 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

54 but the Database was initialized (incorrectly) with writeable=True. 

55 """ 

56 try: 

57 with database.declareStaticTables(create=True) as context: 

58 table = context.addTable( 

59 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

60 ) 

61 # Drop created table so that schema remains empty. 

62 database._metadata.drop_all(database._engine, tables=[table]) 

63 return True 

64 except Exception: 

65 return False 

66 

67 

68class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

69 """Tests for `SqliteDatabase` using a standard file-based database.""" 

70 

71 def setUp(self): 

72 self.root = makeTestTempDir(TESTDIR) 

73 

74 def tearDown(self): 

75 removeTestTempDir(self.root) 

76 

77 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

78 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

79 engine = SqliteDatabase.makeEngine(filename=filename) 

80 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

81 

82 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

83 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

84 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

85 

86 @contextmanager 

87 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

88 with removeWritePermission(database.filename): 

89 yield self.getNewConnection(database, writeable=False) 

90 

91 def testConnection(self): 

92 """Test that different ways of connecting to a SQLite database 

93 are equivalent. 

94 """ 

95 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

96 # Create a read-write database by passing in the filename. 

97 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

98 self.assertEqual(rwFromFilename.filename, filename) 

99 self.assertEqual(rwFromFilename.origin, 0) 

100 self.assertTrue(rwFromFilename.isWriteable()) 

101 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

102 # Create a read-write database via a URI. 

103 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

104 self.assertEqual(rwFromUri.filename, filename) 

105 self.assertEqual(rwFromUri.origin, 0) 

106 self.assertTrue(rwFromUri.isWriteable()) 

107 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

108 # We don't support SQLite URIs inside SQLAlchemy URIs. 

109 with self.assertRaises(NotImplementedError): 

110 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

111 

112 # Test read-only connections against a read-only file. 

113 with removeWritePermission(filename): 

114 # Create a read-only database by passing in the filename. 

115 roFromFilename = SqliteDatabase.fromEngine( 

116 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False 

117 ) 

118 self.assertEqual(roFromFilename.filename, filename) 

119 self.assertEqual(roFromFilename.origin, 0) 

120 self.assertFalse(roFromFilename.isWriteable()) 

121 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

122 # Create a read-write database via a URI. 

123 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

124 self.assertEqual(roFromUri.filename, filename) 

125 self.assertEqual(roFromUri.origin, 0) 

126 self.assertFalse(roFromUri.isWriteable()) 

127 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

128 

129 def testTransactionLocking(self): 

130 # This (inherited) test can't run on SQLite because of our use of an 

131 # aggressive locking strategy there. 

132 pass 

133 

134 

135class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

136 """Tests for `SqliteDatabase` using an in-memory database.""" 

137 

138 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

139 engine = SqliteDatabase.makeEngine(filename=None) 

140 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

141 

142 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

143 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable) 

144 

145 @contextmanager 

146 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

147 yield self.getNewConnection(database, writeable=False) 

148 

149 def testConnection(self): 

150 """Test that different ways of connecting to a SQLite database 

151 are equivalent. 

152 """ 

153 # Create an in-memory database by passing filename=None. 

154 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

155 self.assertIsNone(memFromFilename.filename) 

156 self.assertEqual(memFromFilename.origin, 0) 

157 self.assertTrue(memFromFilename.isWriteable()) 

158 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

159 # Create an in-memory database via a URI. 

160 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

161 self.assertIsNone(memFromUri.filename) 

162 self.assertEqual(memFromUri.origin, 0) 

163 self.assertTrue(memFromUri.isWriteable()) 

164 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

165 # We don't support SQLite URIs inside SQLAlchemy URIs. 

166 with self.assertRaises(NotImplementedError): 

167 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

168 # We don't support read-only in-memory databases. 

169 with self.assertRaises(NotImplementedError): 

170 SqliteDatabase.makeEngine(filename=None, writeable=False) 

171 

172 def testTransactionLocking(self): 

173 # This (inherited) test can't run on SQLite because of our use of an 

174 # aggressive locking strategy there. 

175 pass 

176 

177 

178class SqliteFileRegistryTests(RegistryTests): 

179 """Tests for `Registry` backed by a SQLite file-based database. 

180 

181 Note 

182 ---- 

183 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

184 defines methods that override `unittest.TestCase` methods. To make this 

185 work sublasses have to have this class first in the bases list. 

186 """ 

187 

188 def setUp(self): 

189 self.root = makeTestTempDir(TESTDIR) 

190 

191 def tearDown(self): 

192 removeTestTempDir(self.root) 

193 

194 @classmethod 

195 def getDataDir(cls) -> str: 

196 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

197 

198 def makeRegistry(self) -> Registry: 

199 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

200 config = self.makeRegistryConfig() 

201 config["db"] = f"sqlite:///{filename}" 

202 return Registry.createFromConfig(config, butlerRoot=self.root) 

203 

204 

205class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

206 """Tests for `Registry` backed by a SQLite file-based database. 

207 

208 This test case uses NameKeyCollectionManager and 

209 ByDimensionsDatasetRecordStorageManager. 

210 """ 

211 

212 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

213 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

214 

215 

216class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

217 """Tests for `Registry` backed by a SQLite file-based database. 

218 

219 This test case uses SynthIntKeyCollectionManager and 

220 ByDimensionsDatasetRecordStorageManager. 

221 """ 

222 

223 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

224 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

225 

226 

227class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

228 """Tests for `Registry` backed by a SQLite file-based database. 

229 

230 This test case uses NameKeyCollectionManager and 

231 ByDimensionsDatasetRecordStorageManagerUUID. 

232 """ 

233 

234 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

235 datasetsManager = ( 

236 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

237 ) 

238 

239 

240class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

241 """Tests for `Registry` backed by a SQLite file-based database. 

242 

243 This test case uses SynthIntKeyCollectionManager and 

244 ByDimensionsDatasetRecordStorageManagerUUID. 

245 """ 

246 

247 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

248 datasetsManager = ( 

249 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

250 ) 

251 

252 

253class SqliteMemoryRegistryTests(RegistryTests): 

254 """Tests for `Registry` backed by a SQLite in-memory database.""" 

255 

256 @classmethod 

257 def getDataDir(cls) -> str: 

258 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

259 

260 def makeRegistry(self) -> Registry: 

261 config = self.makeRegistryConfig() 

262 config["db"] = "sqlite://" 

263 return Registry.createFromConfig(config) 

264 

265 def testMissingAttributes(self): 

266 """Test for instantiating a registry against outdated schema which 

267 misses butler_attributes table. 

268 """ 

269 # TODO: Once we have stable gen3 schema everywhere this test can be 

270 # dropped (DM-27373). 

271 config = self.makeRegistryConfig() 

272 config["db"] = "sqlite://" 

273 with self.assertRaises(sqlalchemy.exc.OperationalError): 

274 Registry.fromConfig(config) 

275 

276 

277class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

278 """Tests for `Registry` backed by a SQLite in-memory database. 

279 

280 This test case uses NameKeyCollectionManager and 

281 ByDimensionsDatasetRecordStorageManager. 

282 """ 

283 

284 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

285 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

286 

287 

288class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

289 """Tests for `Registry` backed by a SQLite in-memory database. 

290 

291 This test case uses SynthIntKeyCollectionManager and 

292 ByDimensionsDatasetRecordStorageManager. 

293 """ 

294 

295 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

296 datasetsManager = "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

297 

298 

299class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

300 """Tests for `Registry` backed by a SQLite in-memory database. 

301 

302 This test case uses NameKeyCollectionManager and 

303 ByDimensionsDatasetRecordStorageManagerUUID. 

304 """ 

305 

306 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

307 datasetsManager = ( 

308 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

309 ) 

310 

311 

312class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

313 """Tests for `Registry` backed by a SQLite in-memory database. 

314 

315 This test case uses SynthIntKeyCollectionManager and 

316 ByDimensionsDatasetRecordStorageManagerUUID. 

317 """ 

318 

319 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

320 datasetsManager = ( 

321 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

322 ) 

323 

324 

325if __name__ == "__main__": 325 ↛ 326line 325 didn't jump to line 326, because the condition on line 325 was never true

326 unittest.main()