Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import itertools 

24import os 

25import os.path 

26import shutil 

27import stat 

28import tempfile 

29import unittest 

30 

31import sqlalchemy 

32 

33from lsst.sphgeom import ConvexPolygon, UnitVector3d 

34 

35from lsst.daf.butler import ddl 

36from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

37from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

38from lsst.daf.butler.registry import Registry 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43@contextmanager 

44def removeWritePermission(filename): 

45 mode = os.stat(filename).st_mode 

46 try: 

47 os.chmod(filename, stat.S_IREAD) 

48 yield 

49 finally: 

50 os.chmod(filename, mode) 

51 

52 

53def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

54 """Check whether we really can modify a database. 

55 

56 This intentionally allows any exception to be raised (not just 

57 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

58 but the Database was initialized (incorrectly) with writeable=True. 

59 """ 

60 try: 

61 with database.declareStaticTables(create=True) as context: 

62 table = context.addTable( 

63 "a", 

64 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

65 ) 

66 # Drop created table so that schema remains empty. 

67 database._metadata.drop_all(database._connection, tables=[table]) 

68 return True 

69 except Exception: 

70 return False 

71 

72 

73class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

74 """Tests for `SqliteDatabase` using a standard file-based database. 

75 """ 

76 

77 def setUp(self): 

78 self.root = tempfile.mkdtemp(dir=TESTDIR) 

79 

80 def tearDown(self): 

81 if self.root is not None and os.path.exists(self.root): 

82 shutil.rmtree(self.root, ignore_errors=True) 

83 

84 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

85 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

86 connection = SqliteDatabase.connect(filename=filename) 

87 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

88 

89 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

90 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable) 

91 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection, 

92 writeable=writeable) 

93 

94 @contextmanager 

95 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

96 with removeWritePermission(database.filename): 

97 yield self.getNewConnection(database, writeable=False) 

98 

99 def testConnection(self): 

100 """Test that different ways of connecting to a SQLite database 

101 are equivalent. 

102 """ 

103 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

104 # Create a read-write database by passing in the filename. 

105 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0) 

106 self.assertEqual(rwFromFilename.filename, filename) 

107 self.assertEqual(rwFromFilename.origin, 0) 

108 self.assertTrue(rwFromFilename.isWriteable()) 

109 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

110 # Create a read-write database via a URI. 

111 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

112 self.assertEqual(rwFromUri.filename, filename) 

113 self.assertEqual(rwFromUri.origin, 0) 

114 self.assertTrue(rwFromUri.isWriteable()) 

115 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

116 # We don't support SQLite URIs inside SQLAlchemy URIs. 

117 with self.assertRaises(NotImplementedError): 

118 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true") 

119 

120 # Test read-only connections against a read-only file. 

121 with removeWritePermission(filename): 

122 # Create a read-only database by passing in the filename. 

123 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), 

124 origin=0, writeable=False) 

125 self.assertEqual(roFromFilename.filename, filename) 

126 self.assertEqual(roFromFilename.origin, 0) 

127 self.assertFalse(roFromFilename.isWriteable()) 

128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

129 # Create a read-write database via a URI. 

130 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

131 self.assertEqual(roFromUri.filename, filename) 

132 self.assertEqual(roFromUri.origin, 0) 

133 self.assertFalse(roFromUri.isWriteable()) 

134 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

135 

136 def testTransactionLocking(self): 

137 # This (inherited) test can't run on SQLite because of our use of an 

138 # aggressive locking strategy there. 

139 pass 

140 

141 

142class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

143 """Tests for `SqliteDatabase` using an in-memory database. 

144 """ 

145 

146 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

147 connection = SqliteDatabase.connect(filename=None) 

148 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

149 

150 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

151 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection, 

152 writeable=writeable) 

153 

154 @contextmanager 

155 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

156 yield self.getNewConnection(database, writeable=False) 

157 

158 def testConnection(self): 

159 """Test that different ways of connecting to a SQLite database 

160 are equivalent. 

161 """ 

162 # Create an in-memory database by passing filename=None. 

163 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0) 

164 self.assertIsNone(memFromFilename.filename) 

165 self.assertEqual(memFromFilename.origin, 0) 

166 self.assertTrue(memFromFilename.isWriteable()) 

167 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

168 # Create an in-memory database via a URI. 

169 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

170 self.assertIsNone(memFromUri.filename) 

171 self.assertEqual(memFromUri.origin, 0) 

172 self.assertTrue(memFromUri.isWriteable()) 

173 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

174 # We don't support SQLite URIs inside SQLAlchemy URIs. 

175 with self.assertRaises(NotImplementedError): 

176 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true") 

177 # We don't support read-only in-memory databases. 

178 with self.assertRaises(NotImplementedError): 

179 SqliteDatabase.connect(filename=None, writeable=False) 

180 

181 def testTransactionLocking(self): 

182 # This (inherited) test can't run on SQLite because of our use of an 

183 # aggressive locking strategy there. 

184 pass 

185 

186 

187class SqliteFileRegistryTests(RegistryTests): 

188 """Tests for `Registry` backed by a SQLite file-based database. 

189 

190 Note 

191 ---- 

192 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

193 defines methods that override `unittest.TestCase` methods. To make this 

194 work sublasses have to have this class first in the bases list. 

195 """ 

196 

197 def setUp(self): 

198 self.root = tempfile.mkdtemp(dir=TESTDIR) 

199 

200 def tearDown(self): 

201 if self.root is not None and os.path.exists(self.root): 

202 shutil.rmtree(self.root, ignore_errors=True) 

203 

204 @classmethod 

205 def getDataDir(cls) -> str: 

206 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

207 

208 def makeRegistry(self) -> Registry: 

209 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

210 config = self.makeRegistryConfig() 

211 config["db"] = f"sqlite:///{filename}" 

212 return Registry.fromConfig(config, create=True, butlerRoot=self.root) 

213 

214 

215class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

216 """Tests for `Registry` backed by a SQLite file-based database. 

217 

218 This test case uses NameKeyCollectionManager. 

219 """ 

220 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

221 

222 

223class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

224 """Tests for `Registry` backed by a SQLite file-based database. 

225 

226 This test case uses SynthIntKeyCollectionManager. 

227 """ 

228 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

229 

230 

231class SqliteMemoryRegistryTests(RegistryTests): 

232 """Tests for `Registry` backed by a SQLite in-memory database. 

233 """ 

234 

235 @classmethod 

236 def getDataDir(cls) -> str: 

237 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

238 

239 def makeRegistry(self) -> Registry: 

240 config = self.makeRegistryConfig() 

241 config["db"] = "sqlite://" 

242 return Registry.fromConfig(config, create=True) 

243 

244 def testRegions(self): 

245 """Tests for using region fields in `Registry` dimensions. 

246 """ 

247 # TODO: the test regions used here are enormous (significant fractions 

248 # of the sphere), and that makes this test prohibitively slow on 

249 # most real databases. These should be made more realistic, and the 

250 # test moved to daf/butler/registry/tests/registry.py. 

251 registry = self.makeRegistry() 

252 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0), 

253 UnitVector3d(0, 1, 0), 

254 UnitVector3d(0, 0, 1))) 

255 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0), 

256 UnitVector3d(0, 1, 0), 

257 UnitVector3d(0, 0, 1))) 

258 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0), 

259 UnitVector3d(0, 1, 1), 

260 UnitVector3d(0, 0, 1))) 

261 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0), 

262 UnitVector3d(0, 1, 0), 

263 UnitVector3d(0, 1, 1))) 

264 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2): 

265 self.assertNotEqual(a, b) 

266 

267 # This depends on current dimensions.yaml definitions 

268 self.assertEqual(len(list(registry.queryDataIds(["patch", "htm7"]))), 0) 

269 

270 # Add some dimension entries 

271 registry.insertDimensionData("instrument", {"name": "DummyCam"}) 

272 registry.insertDimensionData("physical_filter", 

273 {"instrument": "DummyCam", "name": "dummy_r", "band": "r"}, 

274 {"instrument": "DummyCam", "name": "dummy_i", "band": "i"}) 

275 for detector in (1, 2, 3, 4, 5): 

276 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector, 

277 "full_name": str(detector)}) 

278 registry.insertDimensionData("visit", 

279 {"instrument": "DummyCam", "id": 0, "name": "zero", 

280 "physical_filter": "dummy_r", "region": regionVisit}, 

281 {"instrument": "DummyCam", "id": 1, "name": "one", 

282 "physical_filter": "dummy_i"}) 

283 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()}) 

284 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract}) 

285 registry.insertDimensionData("patch", 

286 {"skymap": "DummySkyMap", "tract": 0, "patch": 0, 

287 "cell_x": 0, "cell_y": 0, "region": regionPatch}) 

288 registry.insertDimensionData("visit_detector_region", 

289 {"instrument": "DummyCam", "visit": 0, "detector": 2, 

290 "region": regionVisitDetector}) 

291 

292 def getRegion(dataId): 

293 return registry.expandDataId(dataId).region 

294 

295 # Get region for a tract 

296 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0})) 

297 # Attempt to get region for a non-existent tract 

298 with self.assertRaises(LookupError): 

299 getRegion({"skymap": "DummySkyMap", "tract": 1}) 

300 # Get region for a (tract, patch) combination 

301 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0})) 

302 # Get region for a non-existent (tract, patch) combination 

303 with self.assertRaises(LookupError): 

304 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1}) 

305 # Get region for a visit 

306 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0})) 

307 # Attempt to get region for a non-existent visit 

308 with self.assertRaises(LookupError): 

309 getRegion({"instrument": "DummyCam", "visit": 10}) 

310 # Get region for a (visit, detector) combination 

311 self.assertEqual(regionVisitDetector, 

312 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2})) 

313 # Attempt to get region for a non-existent (visit, detector) 

314 # combination. This returns None rather than raising because we don't 

315 # want to require the region record to be present. 

316 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3})) 

317 # getRegion for a dataId containing no spatial dimensions should 

318 # return None 

319 self.assertIsNone(getRegion({"instrument": "DummyCam"})) 

320 # Expanding a data ID with a mix of spatial dimensions should not fail, 

321 # but it may not be implemented, so we don't test that we can get the 

322 # region. 

323 registry.expandDataId({"instrument": "DummyCam", "visit": 0, "detector": 2, 

324 "skymap": "DummySkyMap", "tract": 0}) 

325 # Check if we can get the region for a skypix 

326 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon) 

327 # patch_htm7_overlap should not be empty 

328 self.assertNotEqual(len(list(registry.queryDataIds(["patch", "htm7"]))), 0) 

329 

330 

331class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

332 """Tests for `Registry` backed by a SQLite in-memory database. 

333 

334 This test case uses NameKeyCollectionManager. 

335 """ 

336 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

337 

338 

339class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

340 """Tests for `Registry` backed by a SQLite in-memory database. 

341 

342 This test case uses SynthIntKeyCollectionManager. 

343 """ 

344 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

345 

346 

347if __name__ == "__main__": 347 ↛ 348line 347 didn't jump to line 348, because the condition on line 347 was never true

348 unittest.main()