Coverage for tests/test_sqlite.py: 39%

144 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:26 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import os.path 

24import stat 

25import tempfile 

26import unittest 

27from contextlib import contextmanager 

28 

29import sqlalchemy 

30from lsst.daf.butler import ddl 

31from lsst.daf.butler.registry import _ButlerRegistry, _RegistryFactory 

32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

33from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

34from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39@contextmanager 

40def removeWritePermission(filename): 

41 """Remove the write permission on a file.""" 

42 mode = os.stat(filename).st_mode 

43 try: 

44 os.chmod(filename, stat.S_IREAD) 

45 yield 

46 finally: 

47 os.chmod(filename, mode) 

48 

49 

50def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

51 """Check whether we really can modify a database. 

52 

53 This intentionally allows any exception to be raised (not just 

54 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

55 but the Database was initialized (incorrectly) with writeable=True. 

56 """ 

57 try: 

58 with database.declareStaticTables(create=True) as context: 

59 table = context.addTable( 

60 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

61 ) 

62 # Drop created table so that schema remains empty. 

63 database._metadata.drop_all(database._engine, tables=[table]) 

64 return True 

65 except Exception: 

66 return False 

67 

68 

69class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

70 """Tests for `SqliteDatabase` using a standard file-based database.""" 

71 

72 def setUp(self): 

73 self.root = makeTestTempDir(TESTDIR) 

74 

75 def tearDown(self): 

76 removeTestTempDir(self.root) 

77 

78 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

79 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

80 engine = SqliteDatabase.makeEngine(filename=filename) 

81 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

82 

83 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

84 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

85 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

86 

87 @contextmanager 

88 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

89 with removeWritePermission(database.filename): 

90 yield self.getNewConnection(database, writeable=False) 

91 

92 def testConnection(self): 

93 """Test that different ways of connecting to a SQLite database 

94 are equivalent. 

95 """ 

96 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

97 # Create a read-write database by passing in the filename. 

98 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

99 self.assertEqual(rwFromFilename.filename, filename) 

100 self.assertEqual(rwFromFilename.origin, 0) 

101 self.assertTrue(rwFromFilename.isWriteable()) 

102 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

103 # Create a read-write database via a URI. 

104 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

105 self.assertEqual(rwFromUri.filename, filename) 

106 self.assertEqual(rwFromUri.origin, 0) 

107 self.assertTrue(rwFromUri.isWriteable()) 

108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

109 # We don't support SQLite URIs inside SQLAlchemy URIs. 

110 with self.assertRaises(NotImplementedError): 

111 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

112 

113 # Test read-only connections against a read-only file. 

114 with removeWritePermission(filename): 

115 # Create a read-only database by passing in the filename. 

116 roFromFilename = SqliteDatabase.fromEngine( 

117 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False 

118 ) 

119 self.assertEqual(roFromFilename.filename, filename) 

120 self.assertEqual(roFromFilename.origin, 0) 

121 self.assertFalse(roFromFilename.isWriteable()) 

122 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

123 # Create a read-write database via a URI. 

124 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

125 self.assertEqual(roFromUri.filename, filename) 

126 self.assertEqual(roFromUri.origin, 0) 

127 self.assertFalse(roFromUri.isWriteable()) 

128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

129 

130 def testTransactionLocking(self): 

131 # This (inherited) test can't run on SQLite because of our use of an 

132 # aggressive locking strategy there. 

133 pass 

134 

135 

136class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

137 """Tests for `SqliteDatabase` using an in-memory database.""" 

138 

139 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

140 engine = SqliteDatabase.makeEngine(filename=None) 

141 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

142 

143 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

144 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable) 

145 

146 @contextmanager 

147 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

148 yield self.getNewConnection(database, writeable=False) 

149 

150 def testConnection(self): 

151 """Test that different ways of connecting to a SQLite database 

152 are equivalent. 

153 """ 

154 # Create an in-memory database by passing filename=None. 

155 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

156 self.assertIsNone(memFromFilename.filename) 

157 self.assertEqual(memFromFilename.origin, 0) 

158 self.assertTrue(memFromFilename.isWriteable()) 

159 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

160 # Create an in-memory database via a URI. 

161 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

162 self.assertIsNone(memFromUri.filename) 

163 self.assertEqual(memFromUri.origin, 0) 

164 self.assertTrue(memFromUri.isWriteable()) 

165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

166 # We don't support SQLite URIs inside SQLAlchemy URIs. 

167 with self.assertRaises(NotImplementedError): 

168 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

169 # We don't support read-only in-memory databases. 

170 with self.assertRaises(NotImplementedError): 

171 SqliteDatabase.makeEngine(filename=None, writeable=False) 

172 

173 def testTransactionLocking(self): 

174 # This (inherited) test can't run on SQLite because of our use of an 

175 # aggressive locking strategy there. 

176 pass 

177 

178 

179class SqliteFileRegistryTests(RegistryTests): 

180 """Tests for `Registry` backed by a SQLite file-based database. 

181 

182 Notes 

183 ----- 

184 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

185 defines methods that override `unittest.TestCase` methods. To make this 

186 work sublasses have to have this class first in the bases list. 

187 """ 

188 

189 def setUp(self): 

190 self.root = makeTestTempDir(TESTDIR) 

191 

192 def tearDown(self): 

193 removeTestTempDir(self.root) 

194 

195 @classmethod 

196 def getDataDir(cls) -> str: 

197 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

198 

199 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry: 

200 if share_repo_with is None: 

201 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

202 else: 

203 filename = share_repo_with._db.filename 

204 config = self.makeRegistryConfig() 

205 config["db"] = f"sqlite:///{filename}" 

206 if share_repo_with is None: 

207 return _RegistryFactory(config).create_from_config(butlerRoot=self.root) 

208 else: 

209 return _RegistryFactory(config).from_config(butlerRoot=self.root) 

210 

211 

212class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

213 """Tests for `Registry` backed by a SQLite file-based database. 

214 

215 This test case uses NameKeyCollectionManager and 

216 ByDimensionsDatasetRecordStorageManagerUUID. 

217 """ 

218 

219 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

220 datasetsManager = ( 

221 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

222 ) 

223 

224 

225class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

226 """Tests for `Registry` backed by a SQLite file-based database. 

227 

228 This test case uses SynthIntKeyCollectionManager and 

229 ByDimensionsDatasetRecordStorageManagerUUID. 

230 """ 

231 

232 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

233 datasetsManager = ( 

234 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

235 ) 

236 

237 

238class SqliteMemoryRegistryTests(RegistryTests): 

239 """Tests for `Registry` backed by a SQLite in-memory database.""" 

240 

241 @classmethod 

242 def getDataDir(cls) -> str: 

243 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

244 

245 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry | None: 

246 if share_repo_with is not None: 

247 return None 

248 config = self.makeRegistryConfig() 

249 config["db"] = "sqlite://" 

250 return _RegistryFactory(config).create_from_config() 

251 

252 def testMissingAttributes(self): 

253 """Test for instantiating a registry against outdated schema which 

254 misses butler_attributes table. 

255 """ 

256 # TODO: Once we have stable gen3 schema everywhere this test can be 

257 # dropped (DM-27373). 

258 config = self.makeRegistryConfig() 

259 config["db"] = "sqlite://" 

260 with self.assertRaises(LookupError): 

261 _RegistryFactory(config).from_config() 

262 

263 

264class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

265 """Tests for `Registry` backed by a SQLite in-memory database. 

266 

267 This test case uses NameKeyCollectionManager and 

268 ByDimensionsDatasetRecordStorageManagerUUID. 

269 """ 

270 

271 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

272 datasetsManager = ( 

273 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

274 ) 

275 

276 

277class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

278 """Tests for `Registry` backed by a SQLite in-memory database. 

279 

280 This test case uses SynthIntKeyCollectionManager and 

281 ByDimensionsDatasetRecordStorageManagerUUID. 

282 """ 

283 

284 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

285 datasetsManager = ( 

286 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

287 ) 

288 

289 

290class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

291 """Tests for `Registry` backed by a SQLite in-memory database. 

292 

293 This test case uses version schema with ingest_date as nanoseconds instead 

294 of DATETIME. This tests can be removed when/if we switch to nanoseconds as 

295 default. 

296 """ 

297 

298 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

299 datasetsManager = { 

300 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID", 

301 "schema_version": "2.0.0", 

302 } 

303 

304 

305if __name__ == "__main__": 

306 unittest.main()