Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import itertools 

24import os 

25import os.path 

26import shutil 

27import stat 

28import tempfile 

29import unittest 

30 

31import sqlalchemy 

32 

33from lsst.sphgeom import ConvexPolygon, UnitVector3d 

34 

35from lsst.daf.butler import ddl 

36from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

37from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

38from lsst.daf.butler.registry import Registry 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43@contextmanager 

44def removeWritePermission(filename): 

45 mode = os.stat(filename).st_mode 

46 try: 

47 os.chmod(filename, stat.S_IREAD) 

48 yield 

49 finally: 

50 os.chmod(filename, mode) 

51 

52 

53def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

54 """Check whether we really can modify a database. 

55 

56 This intentionally allows any exception to be raised (not just 

57 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

58 but the Database was initialized (incorrectly) with writeable=True. 

59 """ 

60 try: 

61 with database.declareStaticTables(create=True) as context: 

62 context.addTable( 

63 "a", 

64 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

65 ) 

66 return True 

67 except Exception: 

68 return False 

69 

70 

71class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

72 """Tests for `SqliteDatabase` using a standard file-based database. 

73 """ 

74 

75 def setUp(self): 

76 self.root = tempfile.mkdtemp(dir=TESTDIR) 

77 

78 def tearDown(self): 

79 if self.root is not None and os.path.exists(self.root): 

80 shutil.rmtree(self.root, ignore_errors=True) 

81 

82 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

83 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

84 connection = SqliteDatabase.connect(filename=filename) 

85 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

86 

87 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

88 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable) 

89 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection, 

90 writeable=writeable) 

91 

92 @contextmanager 

93 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

94 with removeWritePermission(database.filename): 

95 yield self.getNewConnection(database, writeable=False) 

96 

97 def testConnection(self): 

98 """Test that different ways of connecting to a SQLite database 

99 are equivalent. 

100 """ 

101 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

102 # Create a read-write database by passing in the filename. 

103 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0) 

104 self.assertEqual(rwFromFilename.filename, filename) 

105 self.assertEqual(rwFromFilename.origin, 0) 

106 self.assertTrue(rwFromFilename.isWriteable()) 

107 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

108 # Create a read-write database via a URI. 

109 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

110 self.assertEqual(rwFromUri.filename, filename) 

111 self.assertEqual(rwFromUri.origin, 0) 

112 self.assertTrue(rwFromUri.isWriteable()) 

113 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

114 # We don't support SQLite URIs inside SQLAlchemy URIs. 

115 with self.assertRaises(NotImplementedError): 

116 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true") 

117 

118 # Test read-only connections against a read-only file. 

119 with removeWritePermission(filename): 

120 # Create a read-only database by passing in the filename. 

121 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), 

122 origin=0, writeable=False) 

123 self.assertEqual(roFromFilename.filename, filename) 

124 self.assertEqual(roFromFilename.origin, 0) 

125 self.assertFalse(roFromFilename.isWriteable()) 

126 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

127 # Create a read-write database via a URI. 

128 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

129 self.assertEqual(roFromUri.filename, filename) 

130 self.assertEqual(roFromUri.origin, 0) 

131 self.assertFalse(roFromUri.isWriteable()) 

132 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

133 

134 

135class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

136 """Tests for `SqliteDatabase` using an in-memory database. 

137 """ 

138 

139 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

140 connection = SqliteDatabase.connect(filename=None) 

141 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

142 

143 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

144 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection, 

145 writeable=writeable) 

146 

147 @contextmanager 

148 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

149 yield self.getNewConnection(database, writeable=False) 

150 

151 def testConnection(self): 

152 """Test that different ways of connecting to a SQLite database 

153 are equivalent. 

154 """ 

155 # Create an in-memory database by passing filename=None. 

156 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0) 

157 self.assertIsNone(memFromFilename.filename) 

158 self.assertEqual(memFromFilename.origin, 0) 

159 self.assertTrue(memFromFilename.isWriteable()) 

160 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

161 # Create an in-memory database via a URI. 

162 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

163 self.assertIsNone(memFromUri.filename) 

164 self.assertEqual(memFromUri.origin, 0) 

165 self.assertTrue(memFromUri.isWriteable()) 

166 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

167 # We don't support SQLite URIs inside SQLAlchemy URIs. 

168 with self.assertRaises(NotImplementedError): 

169 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true") 

170 # We don't support read-only in-memory databases. 

171 with self.assertRaises(NotImplementedError): 

172 SqliteDatabase.connect(filename=None, writeable=False) 

173 

174 

175class SqliteFileRegistryTests(RegistryTests): 

176 """Tests for `Registry` backed by a SQLite file-based database. 

177 

178 Note 

179 ---- 

180 This is not a subclass of `unittest.TestCase` but to avoid repetition it 

181 defines methods that override `unittest.TestCase` methods. To make this 

182 work sublasses have to have this class first in the bases list. 

183 """ 

184 

185 def setUp(self): 

186 self.root = tempfile.mkdtemp(dir=TESTDIR) 

187 

188 def tearDown(self): 

189 if self.root is not None and os.path.exists(self.root): 

190 shutil.rmtree(self.root, ignore_errors=True) 

191 

192 @classmethod 

193 def getDataDir(cls) -> str: 

194 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

195 

196 def makeRegistry(self) -> Registry: 

197 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

198 config = self.makeRegistryConfig() 

199 config["db"] = f"sqlite:///{filename}" 

200 return Registry.fromConfig(config, create=True, butlerRoot=self.root) 

201 

202 

203class SqliteFileRegistryNameKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

204 """Tests for `Registry` backed by a SQLite file-based database. 

205 

206 This test case uses NameKeyCollectionManager. 

207 """ 

208 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

209 

210 

211class SqliteFileRegistrySynthIntKeyCollMgrTestCase(SqliteFileRegistryTests, unittest.TestCase): 

212 """Tests for `Registry` backed by a SQLite file-based database. 

213 

214 This test case uses SynthIntKeyCollectionManager. 

215 """ 

216 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

217 

218 

219class SqliteMemoryRegistryTests(RegistryTests): 

220 """Tests for `Registry` backed by a SQLite in-memory database. 

221 """ 

222 

223 @classmethod 

224 def getDataDir(cls) -> str: 

225 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

226 

227 def makeRegistry(self) -> Registry: 

228 config = self.makeRegistryConfig() 

229 config["db"] = f"sqlite://" 

230 return Registry.fromConfig(config, create=True) 

231 

232 def testRegions(self): 

233 """Tests for using region fields in `Registry` dimensions. 

234 """ 

235 # TODO: the test regions used here are enormous (significant fractions 

236 # of the sphere), and that makes this test prohibitively slow on 

237 # most real databases. These should be made more realistic, and the 

238 # test moved to daf/butler/registry/tests/registry.py. 

239 registry = self.makeRegistry() 

240 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0), 

241 UnitVector3d(0, 1, 0), 

242 UnitVector3d(0, 0, 1))) 

243 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0), 

244 UnitVector3d(0, 1, 0), 

245 UnitVector3d(0, 0, 1))) 

246 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0), 

247 UnitVector3d(0, 1, 1), 

248 UnitVector3d(0, 0, 1))) 

249 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0), 

250 UnitVector3d(0, 1, 0), 

251 UnitVector3d(0, 1, 1))) 

252 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2): 

253 self.assertNotEqual(a, b) 

254 

255 # This depends on current dimensions.yaml definitions 

256 self.assertEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0) 

257 

258 # Add some dimension entries 

259 registry.insertDimensionData("instrument", {"name": "DummyCam"}) 

260 registry.insertDimensionData("physical_filter", 

261 {"instrument": "DummyCam", "name": "dummy_r", "abstract_filter": "r"}, 

262 {"instrument": "DummyCam", "name": "dummy_i", "abstract_filter": "i"}) 

263 for detector in (1, 2, 3, 4, 5): 

264 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector, 

265 "full_name": str(detector)}) 

266 registry.insertDimensionData("visit", 

267 {"instrument": "DummyCam", "id": 0, "name": "zero", 

268 "physical_filter": "dummy_r", "region": regionVisit}, 

269 {"instrument": "DummyCam", "id": 1, "name": "one", 

270 "physical_filter": "dummy_i"}) 

271 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()}) 

272 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract}) 

273 registry.insertDimensionData("patch", 

274 {"skymap": "DummySkyMap", "tract": 0, "patch": 0, 

275 "cell_x": 0, "cell_y": 0, "region": regionPatch}) 

276 registry.insertDimensionData("visit_detector_region", 

277 {"instrument": "DummyCam", "visit": 0, "detector": 2, 

278 "region": regionVisitDetector}) 

279 

280 def getRegion(dataId): 

281 return registry.expandDataId(dataId).region 

282 

283 # Get region for a tract 

284 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0})) 

285 # Attempt to get region for a non-existent tract 

286 with self.assertRaises(LookupError): 

287 getRegion({"skymap": "DummySkyMap", "tract": 1}) 

288 # Get region for a (tract, patch) combination 

289 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0})) 

290 # Get region for a non-existent (tract, patch) combination 

291 with self.assertRaises(LookupError): 

292 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1}) 

293 # Get region for a visit 

294 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0})) 

295 # Attempt to get region for a non-existent visit 

296 with self.assertRaises(LookupError): 

297 getRegion({"instrument": "DummyCam", "visit": 10}) 

298 # Get region for a (visit, detector) combination 

299 self.assertEqual(regionVisitDetector, 

300 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2})) 

301 # Attempt to get region for a non-existent (visit, detector) 

302 # combination. This returns None rather than raising because we don't 

303 # want to require the region record to be present. 

304 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3})) 

305 # getRegion for a dataId containing no spatial dimensions should 

306 # return None 

307 self.assertIsNone(getRegion({"instrument": "DummyCam"})) 

308 # getRegion for a mix of spatial dimensions should return 

309 # NotImplemented, at least until we get it implemented. 

310 self.assertIs(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2, 

311 "skymap": "DummySkyMap", "tract": 0}), 

312 NotImplemented) 

313 # Check if we can get the region for a skypix 

314 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon) 

315 # patch_htm7_overlap should not be empty 

316 self.assertNotEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0) 

317 

318 

319class SqliteMemoryRegistryNameKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

320 """Tests for `Registry` backed by a SQLite in-memory database. 

321 

322 This test case uses NameKeyCollectionManager. 

323 """ 

324 collectionsManager = "lsst.daf.butler.registry.collections.nameKey.NameKeyCollectionManager" 

325 

326 

327class SqliteMemoryRegistrySynthIntKeyCollMgrTestCase(unittest.TestCase, SqliteMemoryRegistryTests): 

328 """Tests for `Registry` backed by a SQLite in-memory database. 

329 

330 This test case uses SynthIntKeyCollectionManager. 

331 """ 

332 collectionsManager = "lsst.daf.butler.registry.collections.synthIntKey.SynthIntKeyCollectionManager" 

333 

334 

335if __name__ == "__main__": 335 ↛ 336line 335 didn't jump to line 336, because the condition on line 335 was never true

336 unittest.main()