Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import itertools 

24import os 

25import os.path 

26import shutil 

27import stat 

28import tempfile 

29import unittest 

30 

31import sqlalchemy 

32 

33from lsst.sphgeom import ConvexPolygon, UnitVector3d 

34 

35from lsst.daf.butler import ddl 

36from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

37from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

38from lsst.daf.butler.registry import Registry 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43@contextmanager 

44def removeWritePermission(filename): 

45 mode = os.stat(filename).st_mode 

46 try: 

47 os.chmod(filename, stat.S_IREAD) 

48 yield 

49 finally: 

50 os.chmod(filename, mode) 

51 

52 

53def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

54 """Check whether we really can modify a database. 

55 

56 This intentionally allows any exception to be raised (not just 

57 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

58 but the Database was initialized (incorrectly) with writeable=True. 

59 """ 

60 try: 

61 with database.declareStaticTables(create=True) as context: 

62 table = context.addTable( 

63 "a", 

64 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

65 ) 

66 # Drop created table so that schema remains empty. 

67 database._metadata.drop_all(database._connection, tables=[table]) 

68 return True 

69 except Exception: 

70 return False 

71 

72 

73class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

74 """Tests for `SqliteDatabase` using a standard file-based database. 

75 """ 

76 

77 def setUp(self): 

78 self.root = tempfile.mkdtemp(dir=TESTDIR) 

79 

80 def tearDown(self): 

81 if self.root is not None and os.path.exists(self.root): 

82 shutil.rmtree(self.root, ignore_errors=True) 

83 

84 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

85 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

86 connection = SqliteDatabase.connect(filename=filename) 

87 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

88 

89 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

90 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable) 

91 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection, 

92 writeable=writeable) 

93 

94 @contextmanager 

95 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

96 with removeWritePermission(database.filename): 

97 yield self.getNewConnection(database, writeable=False) 

98 

99 def testConnection(self): 

100 """Test that different ways of connecting to a SQLite database 

101 are equivalent. 

102 """ 

103 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

104 # Create a read-write database by passing in the filename. 

105 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0) 

106 self.assertEqual(rwFromFilename.filename, filename) 

107 self.assertEqual(rwFromFilename.origin, 0) 

108 self.assertTrue(rwFromFilename.isWriteable()) 

109 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

110 # Create a read-write database via a URI. 

111 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

112 self.assertEqual(rwFromUri.filename, filename) 

113 self.assertEqual(rwFromUri.origin, 0) 

114 self.assertTrue(rwFromUri.isWriteable()) 

115 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

116 # We don't support SQLite URIs inside SQLAlchemy URIs. 

117 with self.assertRaises(NotImplementedError): 

118 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true") 

119 

120 # Test read-only connections against a read-only file. 

121 with removeWritePermission(filename): 

122 # Create a read-only database by passing in the filename. 

123 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), 

124 origin=0, writeable=False) 

125 self.assertEqual(roFromFilename.filename, filename) 

126 self.assertEqual(roFromFilename.origin, 0) 

127 self.assertFalse(roFromFilename.isWriteable()) 

128 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

129 # Create a read-write database via a URI. 

130 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

131 self.assertEqual(roFromUri.filename, filename) 

132 self.assertEqual(roFromUri.origin, 0) 

133 self.assertFalse(roFromUri.isWriteable()) 

134 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

135 

136 

137class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

138 """Tests for `SqliteDatabase` using an in-memory database. 

139 """ 

140 

141 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

142 connection = SqliteDatabase.connect(filename=None) 

143 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

144 

145 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

146 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection, 

147 writeable=writeable) 

148 

149 @contextmanager 

150 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

151 yield self.getNewConnection(database, writeable=False) 

152 

153 def testConnection(self): 

154 """Test that different ways of connecting to a SQLite database 

155 are equivalent. 

156 """ 

157 # Create an in-memory database by passing filename=None. 

158 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0) 

159 self.assertIsNone(memFromFilename.filename) 

160 self.assertEqual(memFromFilename.origin, 0) 

161 self.assertTrue(memFromFilename.isWriteable()) 

162 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

163 # Create an in-memory database via a URI. 

164 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

165 self.assertIsNone(memFromUri.filename) 

166 self.assertEqual(memFromUri.origin, 0) 

167 self.assertTrue(memFromUri.isWriteable()) 

168 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

169 # We don't support SQLite URIs inside SQLAlchemy URIs. 

170 with self.assertRaises(NotImplementedError): 

171 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true") 

172 # We don't support read-only in-memory databases. 

173 with self.assertRaises(NotImplementedError): 

174 SqliteDatabase.connect(filename=None, writeable=False) 

175 

176 

177class SqliteFileRegistryTests(RegistryTests): 

178 """Tests for `Registry` backed by a SQLite file-based database. 

179 

180 Note 

181 ---- 

182 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

183 defines methods that override `unittest.TestCase` methods. To make this 

184 work sublasses have to have this class first in the bases list. 

185 """ 

186 

187 def setUp(self): 

188 self.root = tempfile.mkdtemp(dir=TESTDIR) 

189 

190 def tearDown(self): 

191 if self.root is not None and os.path.exists(self.root): 

192 shutil.rmtree(self.root, ignore_errors=True) 

193 

194 @classmethod 

195 def getDataDir(cls) -> str: 

196 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

197 

198 def makeRegistry(self) -> Registry: 

199 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

200 config = self.makeRegistryConfig() 

201 config["db"] = f"sqlite:///{filename}" 

202 return Registry.fromConfig(config, create=True, butlerRoot=self.root) 

203 

204 

205class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

206 """Tests for `Registry` backed by a SQLite file-based database. 

207 

208 This test case uses NameKeyCollectionManager. 

209 """ 

210 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

211 

212 

213class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

214 """Tests for `Registry` backed by a SQLite file-based database. 

215 

216 This test case uses SynthIntKeyCollectionManager. 

217 """ 

218 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

219 

220 

221class SqliteMemoryRegistryTests(RegistryTests): 

222 """Tests for `Registry` backed by a SQLite in-memory database. 

223 """ 

224 

225 @classmethod 

226 def getDataDir(cls) -> str: 

227 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

228 

229 def makeRegistry(self) -> Registry: 

230 config = self.makeRegistryConfig() 

231 config["db"] = "sqlite://" 

232 return Registry.fromConfig(config, create=True) 

233 

234 def testRegions(self): 

235 """Tests for using region fields in `Registry` dimensions. 

236 """ 

237 # TODO: the test regions used here are enormous (significant fractions 

238 # of the sphere), and that makes this test prohibitively slow on 

239 # most real databases. These should be made more realistic, and the 

240 # test moved to daf/butler/registry/tests/registry.py. 

241 registry = self.makeRegistry() 

242 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0), 

243 UnitVector3d(0, 1, 0), 

244 UnitVector3d(0, 0, 1))) 

245 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0), 

246 UnitVector3d(0, 1, 0), 

247 UnitVector3d(0, 0, 1))) 

248 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0), 

249 UnitVector3d(0, 1, 1), 

250 UnitVector3d(0, 0, 1))) 

251 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0), 

252 UnitVector3d(0, 1, 0), 

253 UnitVector3d(0, 1, 1))) 

254 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2): 

255 self.assertNotEqual(a, b) 

256 

257 # This depends on current dimensions.yaml definitions 

258 self.assertEqual(len(list(registry.queryDataIds(["patch", "htm7"]))), 0) 

259 

260 # Add some dimension entries 

261 registry.insertDimensionData("instrument", {"name": "DummyCam"}) 

262 registry.insertDimensionData("physical_filter", 

263 {"instrument": "DummyCam", "name": "dummy_r", "abstract_filter": "r"}, 

264 {"instrument": "DummyCam", "name": "dummy_i", "abstract_filter": "i"}) 

265 for detector in (1, 2, 3, 4, 5): 

266 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector, 

267 "full_name": str(detector)}) 

268 registry.insertDimensionData("visit", 

269 {"instrument": "DummyCam", "id": 0, "name": "zero", 

270 "physical_filter": "dummy_r", "region": regionVisit}, 

271 {"instrument": "DummyCam", "id": 1, "name": "one", 

272 "physical_filter": "dummy_i"}) 

273 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()}) 

274 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract}) 

275 registry.insertDimensionData("patch", 

276 {"skymap": "DummySkyMap", "tract": 0, "patch": 0, 

277 "cell_x": 0, "cell_y": 0, "region": regionPatch}) 

278 registry.insertDimensionData("visit_detector_region", 

279 {"instrument": "DummyCam", "visit": 0, "detector": 2, 

280 "region": regionVisitDetector}) 

281 

282 def getRegion(dataId): 

283 return registry.expandDataId(dataId).region 

284 

285 # Get region for a tract 

286 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0})) 

287 # Attempt to get region for a non-existent tract 

288 with self.assertRaises(LookupError): 

289 getRegion({"skymap": "DummySkyMap", "tract": 1}) 

290 # Get region for a (tract, patch) combination 

291 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0})) 

292 # Get region for a non-existent (tract, patch) combination 

293 with self.assertRaises(LookupError): 

294 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1}) 

295 # Get region for a visit 

296 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0})) 

297 # Attempt to get region for a non-existent visit 

298 with self.assertRaises(LookupError): 

299 getRegion({"instrument": "DummyCam", "visit": 10}) 

300 # Get region for a (visit, detector) combination 

301 self.assertEqual(regionVisitDetector, 

302 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2})) 

303 # Attempt to get region for a non-existent (visit, detector) 

304 # combination. This returns None rather than raising because we don't 

305 # want to require the region record to be present. 

306 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3})) 

307 # getRegion for a dataId containing no spatial dimensions should 

308 # return None 

309 self.assertIsNone(getRegion({"instrument": "DummyCam"})) 

310 # Expanding a data ID with a mix of spatial dimensions should not fail, 

311 # but it may not be implemented, so we don't test that we can get the 

312 # region. 

313 registry.expandDataId({"instrument": "DummyCam", "visit": 0, "detector": 2, 

314 "skymap": "DummySkyMap", "tract": 0}) 

315 # Check if we can get the region for a skypix 

316 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon) 

317 # patch_htm7_overlap should not be empty 

318 self.assertNotEqual(len(list(registry.queryDataIds(["patch", "htm7"]))), 0) 

319 

320 

321class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

322 """Tests for `Registry` backed by a SQLite in-memory database. 

323 

324 This test case uses NameKeyCollectionManager. 

325 """ 

326 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

327 

328 

329class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

330 """Tests for `Registry` backed by a SQLite in-memory database. 

331 

332 This test case uses SynthIntKeyCollectionManager. 

333 """ 

334 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

335 

336 

337if __name__ == "__main__": 337 ↛ 338line 337 didn't jump to line 338, because the condition on line 337 was never true

338 unittest.main()