Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import os 

24import os.path 

25import tempfile 

26import stat 

27import unittest 

28 

29import sqlalchemy 

30 

31from lsst.daf.butler import ddl 

32from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

33from lsst.daf.butler.registry.attributes import MissingAttributesTableError 

34from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

35from lsst.daf.butler.registry import Registry 

36from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

37 

38TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41@contextmanager 

42def removeWritePermission(filename): 

43 mode = os.stat(filename).st_mode 

44 try: 

45 os.chmod(filename, stat.S_IREAD) 

46 yield 

47 finally: 

48 os.chmod(filename, mode) 

49 

50 

51def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

52 """Check whether we really can modify a database. 

53 

54 This intentionally allows any exception to be raised (not just 

55 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

56 but the Database was initialized (incorrectly) with writeable=True. 

57 """ 

58 try: 

59 with database.declareStaticTables(create=True) as context: 

60 table = context.addTable( 

61 "a", 

62 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

63 ) 

64 # Drop created table so that schema remains empty. 

65 database._metadata.drop_all(database._connection, tables=[table]) 

66 return True 

67 except Exception: 

68 return False 

69 

70 

71class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

72 """Tests for `SqliteDatabase` using a standard file-based database. 

73 """ 

74 

75 def setUp(self): 

76 self.root = makeTestTempDir(TESTDIR) 

77 

78 def tearDown(self): 

79 removeTestTempDir(self.root) 

80 

81 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

82 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

83 connection = SqliteDatabase.connect(filename=filename) 

84 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

85 

86 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

87 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable) 

88 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection, 

89 writeable=writeable) 

90 

91 @contextmanager 

92 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

93 with removeWritePermission(database.filename): 

94 yield self.getNewConnection(database, writeable=False) 

95 

96 def testConnection(self): 

97 """Test that different ways of connecting to a SQLite database 

98 are equivalent. 

99 """ 

100 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

101 # Create a read-write database by passing in the filename. 

102 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0) 

103 self.assertEqual(rwFromFilename.filename, filename) 

104 self.assertEqual(rwFromFilename.origin, 0) 

105 self.assertTrue(rwFromFilename.isWriteable()) 

106 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

107 # Create a read-write database via a URI. 

108 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

109 self.assertEqual(rwFromUri.filename, filename) 

110 self.assertEqual(rwFromUri.origin, 0) 

111 self.assertTrue(rwFromUri.isWriteable()) 

112 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

113 # We don't support SQLite URIs inside SQLAlchemy URIs. 

114 with self.assertRaises(NotImplementedError): 

115 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true") 

116 

117 # Test read-only connections against a read-only file. 

118 with removeWritePermission(filename): 

119 # Create a read-only database by passing in the filename. 

120 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), 

121 origin=0, writeable=False) 

122 self.assertEqual(roFromFilename.filename, filename) 

123 self.assertEqual(roFromFilename.origin, 0) 

124 self.assertFalse(roFromFilename.isWriteable()) 

125 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

126 # Create a read-write database via a URI. 

127 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

128 self.assertEqual(roFromUri.filename, filename) 

129 self.assertEqual(roFromUri.origin, 0) 

130 self.assertFalse(roFromUri.isWriteable()) 

131 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

132 

133 def testTransactionLocking(self): 

134 # This (inherited) test can't run on SQLite because of our use of an 

135 # aggressive locking strategy there. 

136 pass 

137 

138 

139class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

140 """Tests for `SqliteDatabase` using an in-memory database. 

141 """ 

142 

143 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

144 connection = SqliteDatabase.connect(filename=None) 

145 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

146 

147 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

148 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection, 

149 writeable=writeable) 

150 

151 @contextmanager 

152 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

153 yield self.getNewConnection(database, writeable=False) 

154 

155 def testConnection(self): 

156 """Test that different ways of connecting to a SQLite database 

157 are equivalent. 

158 """ 

159 # Create an in-memory database by passing filename=None. 

160 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0) 

161 self.assertIsNone(memFromFilename.filename) 

162 self.assertEqual(memFromFilename.origin, 0) 

163 self.assertTrue(memFromFilename.isWriteable()) 

164 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

165 # Create an in-memory database via a URI. 

166 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

167 self.assertIsNone(memFromUri.filename) 

168 self.assertEqual(memFromUri.origin, 0) 

169 self.assertTrue(memFromUri.isWriteable()) 

170 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

171 # We don't support SQLite URIs inside SQLAlchemy URIs. 

172 with self.assertRaises(NotImplementedError): 

173 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true") 

174 # We don't support read-only in-memory databases. 

175 with self.assertRaises(NotImplementedError): 

176 SqliteDatabase.connect(filename=None, writeable=False) 

177 

178 def testTransactionLocking(self): 

179 # This (inherited) test can't run on SQLite because of our use of an 

180 # aggressive locking strategy there. 

181 pass 

182 

183 

184class SqliteFileRegistryTests(RegistryTests): 

185 """Tests for `Registry` backed by a SQLite file-based database. 

186 

187 Note 

188 ---- 

189 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

190 defines methods that override `unittest.TestCase` methods. To make this 

191 work sublasses have to have this class first in the bases list. 

192 """ 

193 

194 def setUp(self): 

195 self.root = makeTestTempDir(TESTDIR) 

196 

197 def tearDown(self): 

198 removeTestTempDir(self.root) 

199 

200 @classmethod 

201 def getDataDir(cls) -> str: 

202 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

203 

204 def makeRegistry(self) -> Registry: 

205 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

206 config = self.makeRegistryConfig() 

207 config["db"] = f"sqlite:///{filename}" 

208 return Registry.createFromConfig(config, butlerRoot=self.root) 

209 

210 

211class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

212 """Tests for `Registry` backed by a SQLite file-based database. 

213 

214 This test case uses NameKeyCollectionManager. 

215 """ 

216 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

217 

218 

219class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

220 """Tests for `Registry` backed by a SQLite file-based database. 

221 

222 This test case uses SynthIntKeyCollectionManager. 

223 """ 

224 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

225 

226 

227class SqliteMemoryRegistryTests(RegistryTests): 

228 """Tests for `Registry` backed by a SQLite in-memory database. 

229 """ 

230 

231 @classmethod 

232 def getDataDir(cls) -> str: 

233 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

234 

235 def makeRegistry(self) -> Registry: 

236 config = self.makeRegistryConfig() 

237 config["db"] = "sqlite://" 

238 return Registry.createFromConfig(config) 

239 

240 def testMissingAttributes(self): 

241 """Test for instantiating a registry against outdated schema which 

242 misses butler_attributes table. 

243 """ 

244 # TODO: Once we have stable gen3 schema everywhere this test can be 

245 # dropped (DM-27373). 

246 config = self.makeRegistryConfig() 

247 config["db"] = "sqlite://" 

248 with self.assertRaises(MissingAttributesTableError): 

249 Registry.fromConfig(config) 

250 

251 

252class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

253 """Tests for `Registry` backed by a SQLite in-memory database. 

254 

255 This test case uses NameKeyCollectionManager. 

256 """ 

257 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

258 

259 

260class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

261 """Tests for `Registry` backed by a SQLite in-memory database. 

262 

263 This test case uses SynthIntKeyCollectionManager. 

264 """ 

265 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

266 

267 

268if __name__ == "__main__": 268 ↛ 269line 268 didn't jump to line 269, because the condition on line 268 was never true

269 unittest.main()