Coverage for tests/test_sqlite.py: 39%

144 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import os 

29import os.path 

30import stat 

31import tempfile 

32import unittest 

33from contextlib import contextmanager 

34 

35import sqlalchemy 

36from lsst.daf.butler import ddl 

37from lsst.daf.butler.registry import _ButlerRegistry, _RegistryFactory 

38from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

39from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

40from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

41 

42TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

43 

44 

45@contextmanager 

46def removeWritePermission(filename): 

47 """Remove the write permission on a file.""" 

48 mode = os.stat(filename).st_mode 

49 try: 

50 os.chmod(filename, stat.S_IREAD) 

51 yield 

52 finally: 

53 os.chmod(filename, mode) 

54 

55 

56def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

57 """Check whether we really can modify a database. 

58 

59 This intentionally allows any exception to be raised (not just 

60 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

61 but the Database was initialized (incorrectly) with writeable=True. 

62 """ 

63 try: 

64 with database.declareStaticTables(create=True) as context: 

65 table = context.addTable( 

66 "a", ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

67 ) 

68 # Drop created table so that schema remains empty. 

69 database._metadata.drop_all(database._engine, tables=[table]) 

70 return True 

71 except Exception: 

72 return False 

73 

74 

75class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

76 """Tests for `SqliteDatabase` using a standard file-based database.""" 

77 

78 def setUp(self): 

79 self.root = makeTestTempDir(TESTDIR) 

80 

81 def tearDown(self): 

82 removeTestTempDir(self.root) 

83 

84 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

85 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

86 engine = SqliteDatabase.makeEngine(filename=filename) 

87 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

88 

89 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

90 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

91 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

92 

93 @contextmanager 

94 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

95 with removeWritePermission(database.filename): 

96 yield self.getNewConnection(database, writeable=False) 

97 

98 def testConnection(self): 

99 """Test that different ways of connecting to a SQLite database 

100 are equivalent. 

101 """ 

102 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

103 # Create a read-write database by passing in the filename. 

104 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

105 self.assertEqual(rwFromFilename.filename, filename) 

106 self.assertEqual(rwFromFilename.origin, 0) 

107 self.assertTrue(rwFromFilename.isWriteable()) 

108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

109 # Create a read-write database via a URI. 

110 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

111 self.assertEqual(rwFromUri.filename, filename) 

112 self.assertEqual(rwFromUri.origin, 0) 

113 self.assertTrue(rwFromUri.isWriteable()) 

114 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

115 # We don't support SQLite URIs inside SQLAlchemy URIs. 

116 with self.assertRaises(NotImplementedError): 

117 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

118 

119 # Test read-only connections against a read-only file. 

120 with removeWritePermission(filename): 

121 # Create a read-only database by passing in the filename. 

122 roFromFilename = SqliteDatabase.fromEngine( 

123 SqliteDatabase.makeEngine(filename=filename), origin=0, writeable=False 

124 ) 

125 self.assertEqual(roFromFilename.filename, filename) 

126 self.assertEqual(roFromFilename.origin, 0) 

127 self.assertFalse(roFromFilename.isWriteable()) 

128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

129 # Create a read-write database via a URI. 

130 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

131 self.assertEqual(roFromUri.filename, filename) 

132 self.assertEqual(roFromUri.origin, 0) 

133 self.assertFalse(roFromUri.isWriteable()) 

134 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

135 

136 def testTransactionLocking(self): 

137 # This (inherited) test can't run on SQLite because of our use of an 

138 # aggressive locking strategy there. 

139 pass 

140 

141 

142class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

143 """Tests for `SqliteDatabase` using an in-memory database.""" 

144 

145 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

146 engine = SqliteDatabase.makeEngine(filename=None) 

147 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

148 

149 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

150 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, writeable=writeable) 

151 

152 @contextmanager 

153 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

154 yield self.getNewConnection(database, writeable=False) 

155 

156 def testConnection(self): 

157 """Test that different ways of connecting to a SQLite database 

158 are equivalent. 

159 """ 

160 # Create an in-memory database by passing filename=None. 

161 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

162 self.assertIsNone(memFromFilename.filename) 

163 self.assertEqual(memFromFilename.origin, 0) 

164 self.assertTrue(memFromFilename.isWriteable()) 

165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

166 # Create an in-memory database via a URI. 

167 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

168 self.assertIsNone(memFromUri.filename) 

169 self.assertEqual(memFromUri.origin, 0) 

170 self.assertTrue(memFromUri.isWriteable()) 

171 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

172 # We don't support SQLite URIs inside SQLAlchemy URIs. 

173 with self.assertRaises(NotImplementedError): 

174 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

175 # We don't support read-only in-memory databases. 

176 with self.assertRaises(NotImplementedError): 

177 SqliteDatabase.makeEngine(filename=None, writeable=False) 

178 

179 def testTransactionLocking(self): 

180 # This (inherited) test can't run on SQLite because of our use of an 

181 # aggressive locking strategy there. 

182 pass 

183 

184 

185class SqliteFileRegistryTests(RegistryTests): 

186 """Tests for `Registry` backed by a SQLite file-based database. 

187 

188 Notes 

189 ----- 

190 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

191 defines methods that override `unittest.TestCase` methods. To make this 

192 work sublasses have to have this class first in the bases list. 

193 """ 

194 

195 def setUp(self): 

196 self.root = makeTestTempDir(TESTDIR) 

197 

198 def tearDown(self): 

199 removeTestTempDir(self.root) 

200 

201 @classmethod 

202 def getDataDir(cls) -> str: 

203 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

204 

205 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry: 

206 if share_repo_with is None: 

207 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

208 else: 

209 filename = share_repo_with._db.filename 

210 config = self.makeRegistryConfig() 

211 config["db"] = f"sqlite:///{filename}" 

212 if share_repo_with is None: 

213 return _RegistryFactory(config).create_from_config(butlerRoot=self.root) 

214 else: 

215 return _RegistryFactory(config).from_config(butlerRoot=self.root) 

216 

217 

218class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

219 """Tests for `Registry` backed by a SQLite file-based database. 

220 

221 This test case uses NameKeyCollectionManager and 

222 ByDimensionsDatasetRecordStorageManagerUUID. 

223 """ 

224 

225 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

226 datasetsManager = ( 

227 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

228 ) 

229 

230 

231class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

232 """Tests for `Registry` backed by a SQLite file-based database. 

233 

234 This test case uses SynthIntKeyCollectionManager and 

235 ByDimensionsDatasetRecordStorageManagerUUID. 

236 """ 

237 

238 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

239 datasetsManager = ( 

240 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

241 ) 

242 

243 

244class SqliteMemoryRegistryTests(RegistryTests): 

245 """Tests for `Registry` backed by a SQLite in-memory database.""" 

246 

247 @classmethod 

248 def getDataDir(cls) -> str: 

249 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

250 

251 def makeRegistry(self, share_repo_with: _ButlerRegistry | None = None) -> _ButlerRegistry | None: 

252 if share_repo_with is not None: 

253 return None 

254 config = self.makeRegistryConfig() 

255 config["db"] = "sqlite://" 

256 return _RegistryFactory(config).create_from_config() 

257 

258 def testMissingAttributes(self): 

259 """Test for instantiating a registry against outdated schema which 

260 misses butler_attributes table. 

261 """ 

262 # TODO: Once we have stable gen3 schema everywhere this test can be 

263 # dropped (DM-27373). 

264 config = self.makeRegistryConfig() 

265 config["db"] = "sqlite://" 

266 with self.assertRaises(LookupError): 

267 _RegistryFactory(config).from_config() 

268 

269 

270class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

271 """Tests for `Registry` backed by a SQLite in-memory database. 

272 

273 This test case uses NameKeyCollectionManager and 

274 ByDimensionsDatasetRecordStorageManagerUUID. 

275 """ 

276 

277 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

278 datasetsManager = ( 

279 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

280 ) 

281 

282 

283class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

284 """Tests for `Registry` backed by a SQLite in-memory database. 

285 

286 This test case uses SynthIntKeyCollectionManager and 

287 ByDimensionsDatasetRecordStorageManagerUUID. 

288 """ 

289 

290 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

291 datasetsManager = ( 

292 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

293 ) 

294 

295 

296class SqliteMemoryRegistryAstropyIngestDateTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

297 """Tests for `Registry` backed by a SQLite in-memory database. 

298 

299 This test case uses version schema with ingest_date as nanoseconds instead 

300 of DATETIME. This tests can be removed when/if we switch to nanoseconds as 

301 default. 

302 """ 

303 

304 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

305 datasetsManager = { 

306 "cls": "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID", 

307 "schema_version": "2.0.0", 

308 } 

309 

310 

311if __name__ == "__main__": 

312 unittest.main()