Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import os 

24import os.path 

25import tempfile 

26import stat 

27import unittest 

28 

29import sqlalchemy 

30 

31from lsst.daf.butler import ddl 

32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

33from lsst.daf.butler.registry.attributes import MissingAttributesTableError 

34from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

35from lsst.daf.butler.registry import Registry 

36from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

37 

38TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41@contextmanager 

42def removeWritePermission(filename): 

43 mode = os.stat(filename).st_mode 

44 try: 

45 os.chmod(filename, stat.S_IREAD) 

46 yield 

47 finally: 

48 os.chmod(filename, mode) 

49 

50 

51def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

52 """Check whether we really can modify a database. 

53 

54 This intentionally allows any exception to be raised (not just 

55 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

56 but the Database was initialized (incorrectly) with writeable=True. 

57 """ 

58 try: 

59 with database.declareStaticTables(create=True) as context: 

60 table = context.addTable( 

61 "a", 

62 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

63 ) 

64 # Drop created table so that schema remains empty. 

65 database._metadata.drop_all(database._engine, tables=[table]) 

66 return True 

67 except Exception: 

68 return False 

69 

70 

71class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

72 """Tests for `SqliteDatabase` using a standard file-based database. 

73 """ 

74 

75 def setUp(self): 

76 self.root = makeTestTempDir(TESTDIR) 

77 

78 def tearDown(self): 

79 removeTestTempDir(self.root) 

80 

81 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

82 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

83 engine = SqliteDatabase.makeEngine(filename=filename) 

84 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

85 

86 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

87 engine = SqliteDatabase.makeEngine(filename=database.filename, writeable=writeable) 

88 return SqliteDatabase.fromEngine(origin=database.origin, engine=engine, writeable=writeable) 

89 

90 @contextmanager 

91 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

92 with removeWritePermission(database.filename): 

93 yield self.getNewConnection(database, writeable=False) 

94 

95 def testConnection(self): 

96 """Test that different ways of connecting to a SQLite database 

97 are equivalent. 

98 """ 

99 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

100 # Create a read-write database by passing in the filename. 

101 rwFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), origin=0) 

102 self.assertEqual(rwFromFilename.filename, filename) 

103 self.assertEqual(rwFromFilename.origin, 0) 

104 self.assertTrue(rwFromFilename.isWriteable()) 

105 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

106 # Create a read-write database via a URI. 

107 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

108 self.assertEqual(rwFromUri.filename, filename) 

109 self.assertEqual(rwFromUri.origin, 0) 

110 self.assertTrue(rwFromUri.isWriteable()) 

111 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

112 # We don't support SQLite URIs inside SQLAlchemy URIs. 

113 with self.assertRaises(NotImplementedError): 

114 SqliteDatabase.makeEngine(uri=f"sqlite:///file:{filename}?uri=true") 

115 

116 # Test read-only connections against a read-only file. 

117 with removeWritePermission(filename): 

118 # Create a read-only database by passing in the filename. 

119 roFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=filename), 

120 origin=0, writeable=False) 

121 self.assertEqual(roFromFilename.filename, filename) 

122 self.assertEqual(roFromFilename.origin, 0) 

123 self.assertFalse(roFromFilename.isWriteable()) 

124 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

125 # Create a read-write database via a URI. 

126 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

127 self.assertEqual(roFromUri.filename, filename) 

128 self.assertEqual(roFromUri.origin, 0) 

129 self.assertFalse(roFromUri.isWriteable()) 

130 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

131 

132 def testTransactionLocking(self): 

133 # This (inherited) test can't run on SQLite because of our use of an 

134 # aggressive locking strategy there. 

135 pass 

136 

137 

138class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

139 """Tests for `SqliteDatabase` using an in-memory database. 

140 """ 

141 

142 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

143 engine = SqliteDatabase.makeEngine(filename=None) 

144 return SqliteDatabase.fromEngine(engine=engine, origin=origin) 

145 

146 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

147 return SqliteDatabase.fromEngine(origin=database.origin, engine=database._engine, 

148 writeable=writeable) 

149 

150 @contextmanager 

151 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

152 yield self.getNewConnection(database, writeable=False) 

153 

154 def testConnection(self): 

155 """Test that different ways of connecting to a SQLite database 

156 are equivalent. 

157 """ 

158 # Create an in-memory database by passing filename=None. 

159 memFromFilename = SqliteDatabase.fromEngine(SqliteDatabase.makeEngine(filename=None), origin=0) 

160 self.assertIsNone(memFromFilename.filename) 

161 self.assertEqual(memFromFilename.origin, 0) 

162 self.assertTrue(memFromFilename.isWriteable()) 

163 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

164 # Create an in-memory database via a URI. 

165 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

166 self.assertIsNone(memFromUri.filename) 

167 self.assertEqual(memFromUri.origin, 0) 

168 self.assertTrue(memFromUri.isWriteable()) 

169 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

170 # We don't support SQLite URIs inside SQLAlchemy URIs. 

171 with self.assertRaises(NotImplementedError): 

172 SqliteDatabase.makeEngine(uri="sqlite:///:memory:?uri=true") 

173 # We don't support read-only in-memory databases. 

174 with self.assertRaises(NotImplementedError): 

175 SqliteDatabase.makeEngine(filename=None, writeable=False) 

176 

177 def testTransactionLocking(self): 

178 # This (inherited) test can't run on SQLite because of our use of an 

179 # aggressive locking strategy there. 

180 pass 

181 

182 

183class SqliteFileRegistryTests(RegistryTests): 

184 """Tests for `Registry` backed by a SQLite file-based database. 

185 

186 Note 

187 ---- 

188 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

189 defines methods that override `unittest.TestCase` methods. To make this 

190 work sublasses have to have this class first in the bases list. 

191 """ 

192 

193 def setUp(self): 

194 self.root = makeTestTempDir(TESTDIR) 

195 

196 def tearDown(self): 

197 removeTestTempDir(self.root) 

198 

199 @classmethod 

200 def getDataDir(cls) -> str: 

201 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

202 

203 def makeRegistry(self) -> Registry: 

204 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

205 config = self.makeRegistryConfig() 

206 config["db"] = f"sqlite:///{filename}" 

207 return Registry.createFromConfig(config, butlerRoot=self.root) 

208 

209 

210class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

211 """Tests for `Registry` backed by a SQLite file-based database. 

212 

213 This test case uses NameKeyCollectionManager and 

214 ByDimensionsDatasetRecordStorageManager. 

215 """ 

216 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

217 datasetsManager = \ 

218 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

219 

220 

221class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

222 """Tests for `Registry` backed by a SQLite file-based database. 

223 

224 This test case uses SynthIntKeyCollectionManager and 

225 ByDimensionsDatasetRecordStorageManager. 

226 """ 

227 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

228 datasetsManager = \ 

229 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

230 

231 

232class SqliteFileRegistryNameKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

233 """Tests for `Registry` backed by a SQLite file-based database. 

234 

235 This test case uses NameKeyCollectionManager and 

236 ByDimensionsDatasetRecordStorageManagerUUID. 

237 """ 

238 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

239 datasetsManager = \ 

240 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

241 

242 

243class SqliteFileRegistrySynthIntKeyCollMgrUUIDTestCase(SqliteFileRegistryTests, unittest.TestCase): 

244 """Tests for `Registry` backed by a SQLite file-based database. 

245 

246 This test case uses SynthIntKeyCollectionManager and 

247 ByDimensionsDatasetRecordStorageManagerUUID. 

248 """ 

249 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

250 datasetsManager = \ 

251 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

252 

253 

254class SqliteMemoryRegistryTests(RegistryTests): 

255 """Tests for `Registry` backed by a SQLite in-memory database. 

256 """ 

257 

258 @classmethod 

259 def getDataDir(cls) -> str: 

260 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

261 

262 def makeRegistry(self) -> Registry: 

263 config = self.makeRegistryConfig() 

264 config["db"] = "sqlite://" 

265 return Registry.createFromConfig(config) 

266 

267 def testMissingAttributes(self): 

268 """Test for instantiating a registry against outdated schema which 

269 misses butler_attributes table. 

270 """ 

271 # TODO: Once we have stable gen3 schema everywhere this test can be 

272 # dropped (DM-27373). 

273 config = self.makeRegistryConfig() 

274 config["db"] = "sqlite://" 

275 with self.assertRaises(MissingAttributesTableError): 

276 Registry.fromConfig(config) 

277 

278 

279class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

280 """Tests for `Registry` backed by a SQLite in-memory database. 

281 

282 This test case uses NameKeyCollectionManager and 

283 ByDimensionsDatasetRecordStorageManager. 

284 """ 

285 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

286 datasetsManager = \ 

287 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

288 

289 

290class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

291 """Tests for `Registry` backed by a SQLite in-memory database. 

292 

293 This test case uses SynthIntKeyCollectionManager and 

294 ByDimensionsDatasetRecordStorageManager. 

295 """ 

296 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

297 datasetsManager = \ 

298 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManager" 

299 

300 

301class SqliteMemoryRegistryNameKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

302 """Tests for `Registry` backed by a SQLite in-memory database. 

303 

304 This test case uses NameKeyCollectionManager and 

305 ByDimensionsDatasetRecordStorageManagerUUID. 

306 """ 

307 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

308 datasetsManager = \ 

309 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

310 

311 

312class SqliteMemoryRegistrySynthIntKeyCollMgrUUIDTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

313 """Tests for `Registry` backed by a SQLite in-memory database. 

314 

315 This test case uses SynthIntKeyCollectionManager and 

316 ByDimensionsDatasetRecordStorageManagerUUID. 

317 """ 

318 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

319 datasetsManager = \ 

320 "lsst.daf.butler.registry.datasets.byDimensions.ByDimensionsDatasetRecordStorageManagerUUID" 

321 

322 

323if __name__ == "__main__": 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true

324 unittest.main()