Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import os 

24import os.path 

25import shutil 

26import stat 

27import tempfile 

28import unittest 

29 

30import sqlalchemy 

31 

32from lsst.daf.butler import ddl 

33from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

34from lsst.daf.butler.registry.attributes import MissingAttributesTableError 

35from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

36from lsst.daf.butler.registry import Registry 

37 

38TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41@contextmanager 

42def removeWritePermission(filename): 

43 mode = os.stat(filename).st_mode 

44 try: 

45 os.chmod(filename, stat.S_IREAD) 

46 yield 

47 finally: 

48 os.chmod(filename, mode) 

49 

50 

51def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

52 """Check whether we really can modify a database. 

53 

54 This intentionally allows any exception to be raised (not just 

55 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

56 but the Database was initialized (incorrectly) with writeable=True. 

57 """ 

58 try: 

59 with database.declareStaticTables(create=True) as context: 

60 table = context.addTable( 

61 "a", 

62 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

63 ) 

64 # Drop created table so that schema remains empty. 

65 database._metadata.drop_all(database._connection, tables=[table]) 

66 return True 

67 except Exception: 

68 return False 

69 

70 

71class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

72 """Tests for `SqliteDatabase` using a standard file-based database. 

73 """ 

74 

75 def setUp(self): 

76 self.root = tempfile.mkdtemp(dir=TESTDIR) 

77 

78 def tearDown(self): 

79 if self.root is not None and os.path.exists(self.root): 

80 shutil.rmtree(self.root, ignore_errors=True) 

81 

82 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

83 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

84 connection = SqliteDatabase.connect(filename=filename) 

85 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

86 

87 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

88 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable) 

89 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection, 

90 writeable=writeable) 

91 

92 @contextmanager 

93 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

94 with removeWritePermission(database.filename): 

95 yield self.getNewConnection(database, writeable=False) 

96 

97 def testConnection(self): 

98 """Test that different ways of connecting to a SQLite database 

99 are equivalent. 

100 """ 

101 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

102 # Create a read-write database by passing in the filename. 

103 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0) 

104 self.assertEqual(rwFromFilename.filename, filename) 

105 self.assertEqual(rwFromFilename.origin, 0) 

106 self.assertTrue(rwFromFilename.isWriteable()) 

107 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

108 # Create a read-write database via a URI. 

109 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

110 self.assertEqual(rwFromUri.filename, filename) 

111 self.assertEqual(rwFromUri.origin, 0) 

112 self.assertTrue(rwFromUri.isWriteable()) 

113 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

114 # We don't support SQLite URIs inside SQLAlchemy URIs. 

115 with self.assertRaises(NotImplementedError): 

116 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true") 

117 

118 # Test read-only connections against a read-only file. 

119 with removeWritePermission(filename): 

120 # Create a read-only database by passing in the filename. 

121 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), 

122 origin=0, writeable=False) 

123 self.assertEqual(roFromFilename.filename, filename) 

124 self.assertEqual(roFromFilename.origin, 0) 

125 self.assertFalse(roFromFilename.isWriteable()) 

126 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

127 # Create a read-write database via a URI. 

128 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

129 self.assertEqual(roFromUri.filename, filename) 

130 self.assertEqual(roFromUri.origin, 0) 

131 self.assertFalse(roFromUri.isWriteable()) 

132 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

133 

134 def testTransactionLocking(self): 

135 # This (inherited) test can't run on SQLite because of our use of an 

136 # aggressive locking strategy there. 

137 pass 

138 

139 

140class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

141 """Tests for `SqliteDatabase` using an in-memory database. 

142 """ 

143 

144 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

145 connection = SqliteDatabase.connect(filename=None) 

146 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

147 

148 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

149 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection, 

150 writeable=writeable) 

151 

152 @contextmanager 

153 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

154 yield self.getNewConnection(database, writeable=False) 

155 

156 def testConnection(self): 

157 """Test that different ways of connecting to a SQLite database 

158 are equivalent. 

159 """ 

160 # Create an in-memory database by passing filename=None. 

161 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0) 

162 self.assertIsNone(memFromFilename.filename) 

163 self.assertEqual(memFromFilename.origin, 0) 

164 self.assertTrue(memFromFilename.isWriteable()) 

165 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

166 # Create an in-memory database via a URI. 

167 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

168 self.assertIsNone(memFromUri.filename) 

169 self.assertEqual(memFromUri.origin, 0) 

170 self.assertTrue(memFromUri.isWriteable()) 

171 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

172 # We don't support SQLite URIs inside SQLAlchemy URIs. 

173 with self.assertRaises(NotImplementedError): 

174 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true") 

175 # We don't support read-only in-memory databases. 

176 with self.assertRaises(NotImplementedError): 

177 SqliteDatabase.connect(filename=None, writeable=False) 

178 

179 def testTransactionLocking(self): 

180 # This (inherited) test can't run on SQLite because of our use of an 

181 # aggressive locking strategy there. 

182 pass 

183 

184 

185class SqliteFileRegistryTests(RegistryTests): 

186 """Tests for `Registry` backed by a SQLite file-based database. 

187 

188 Note 

189 ---- 

190 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

191 defines methods that override `unittest.TestCase` methods. To make this 

192 work sublasses have to have this class first in the bases list. 

193 """ 

194 

195 def setUp(self): 

196 self.root = tempfile.mkdtemp(dir=TESTDIR) 

197 

198 def tearDown(self): 

199 if self.root is not None and os.path.exists(self.root): 

200 shutil.rmtree(self.root, ignore_errors=True) 

201 

202 @classmethod 

203 def getDataDir(cls) -> str: 

204 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

205 

206 def makeRegistry(self) -> Registry: 

207 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

208 config = self.makeRegistryConfig() 

209 config["db"] = f"sqlite:///{filename}" 

210 return Registry.createFromConfig(config, butlerRoot=self.root) 

211 

212 

213class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

214 """Tests for `Registry` backed by a SQLite file-based database. 

215 

216 This test case uses NameKeyCollectionManager. 

217 """ 

218 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

219 

220 

221class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

222 """Tests for `Registry` backed by a SQLite file-based database. 

223 

224 This test case uses SynthIntKeyCollectionManager. 

225 """ 

226 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

227 

228 

229class SqliteMemoryRegistryTests(RegistryTests): 

230 """Tests for `Registry` backed by a SQLite in-memory database. 

231 """ 

232 

233 @classmethod 

234 def getDataDir(cls) -> str: 

235 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

236 

237 def makeRegistry(self) -> Registry: 

238 config = self.makeRegistryConfig() 

239 config["db"] = "sqlite://" 

240 return Registry.createFromConfig(config) 

241 

242 def testMissingAttributes(self): 

243 """Test for instantiating a registry against outdated schema which 

244 misses butler_attributes table. 

245 """ 

246 # TODO: Once we have stable gen3 schema everywhere this test can be 

247 # dropped (DM-27373). 

248 config = self.makeRegistryConfig() 

249 config["db"] = "sqlite://" 

250 with self.assertRaises(MissingAttributesTableError): 

251 Registry.fromConfig(config) 

252 

253 

254class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

255 """Tests for `Registry` backed by a SQLite in-memory database. 

256 

257 This test case uses NameKeyCollectionManager. 

258 """ 

259 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

260 

261 

262class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

263 """Tests for `Registry` backed by a SQLite in-memory database. 

264 

265 This test case uses SynthIntKeyCollectionManager. 

266 """ 

267 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

268 

269 

270if __name__ == "__main__": 270 ↛ 271line 270 didn't jump to line 271, because the condition on line 270 was never true

271 unittest.main()