Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from contextlib import contextmanager 

23import itertools 

24import os 

25import os.path 

26import shutil 

27import stat 

28import tempfile 

29import unittest 

30 

31import sqlalchemy 

32 

33from lsst.sphgeom import ConvexPolygon, UnitVector3d 

34 

35from lsst.daf.butler import ddl 

36from lsst.daf.butler.registry import RegistryConfig 

37from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase 

38from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests 

39from lsst.daf.butler.registry import Registry 

40 

41TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

42 

43 

44@contextmanager 

45def removeWritePermission(filename): 

46 mode = os.stat(filename).st_mode 

47 try: 

48 os.chmod(filename, stat.S_IREAD) 

49 yield 

50 finally: 

51 os.chmod(filename, mode) 

52 

53 

54def isEmptyDatabaseActuallyWriteable(database: SqliteDatabase) -> bool: 

55 """Check whether we really can modify a database. 

56 

57 This intentionally allows any exception to be raised (not just 

58 `ReadOnlyDatabaseError`) to deal with cases where the file is read-only 

59 but the Database was initialized (incorrectly) with writeable=True. 

60 """ 

61 try: 

62 with database.declareStaticTables(create=True) as context: 

63 context.addTable( 

64 "a", 

65 ddl.TableSpec(fields=[ddl.FieldSpec("b", dtype=sqlalchemy.Integer, primaryKey=True)]) 

66 ) 

67 return True 

68 except Exception: 

69 return False 

70 

71 

72class SqliteFileDatabaseTestCase(unittest.TestCase, DatabaseTests): 

73 """Tests for `SqliteDatabase` using a standard file-based database. 

74 """ 

75 

76 def setUp(self): 

77 self.root = tempfile.mkdtemp(dir=TESTDIR) 

78 

79 def tearDown(self): 

80 if self.root is not None and os.path.exists(self.root): 

81 shutil.rmtree(self.root, ignore_errors=True) 

82 

83 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

84 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

85 connection = SqliteDatabase.connect(filename=filename) 

86 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

87 

88 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

89 connection = SqliteDatabase.connect(filename=database.filename, writeable=writeable) 

90 return SqliteDatabase.fromConnection(origin=database.origin, connection=connection, 

91 writeable=writeable) 

92 

93 @contextmanager 

94 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

95 with removeWritePermission(database.filename): 

96 yield self.getNewConnection(database, writeable=False) 

97 

98 def testConnection(self): 

99 """Test that different ways of connecting to a SQLite database 

100 are equivalent. 

101 """ 

102 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

103 # Create a read-write database by passing in the filename. 

104 rwFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), origin=0) 

105 self.assertEqual(rwFromFilename.filename, filename) 

106 self.assertEqual(rwFromFilename.origin, 0) 

107 self.assertTrue(rwFromFilename.isWriteable()) 

108 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromFilename)) 

109 # Create a read-write database via a URI. 

110 rwFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0) 

111 self.assertEqual(rwFromUri.filename, filename) 

112 self.assertEqual(rwFromUri.origin, 0) 

113 self.assertTrue(rwFromUri.isWriteable()) 

114 self.assertTrue(isEmptyDatabaseActuallyWriteable(rwFromUri)) 

115 # We don't support SQLite URIs inside SQLAlchemy URIs. 

116 with self.assertRaises(NotImplementedError): 

117 SqliteDatabase.connect(uri=f"sqlite:///file:{filename}?uri=true") 

118 

119 # Test read-only connections against a read-only file. 

120 with removeWritePermission(filename): 

121 # Create a read-only database by passing in the filename. 

122 roFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=filename), 

123 origin=0, writeable=False) 

124 self.assertEqual(roFromFilename.filename, filename) 

125 self.assertEqual(roFromFilename.origin, 0) 

126 self.assertFalse(roFromFilename.isWriteable()) 

127 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromFilename)) 

128 # Create a read-write database via a URI. 

129 roFromUri = SqliteDatabase.fromUri(f"sqlite:///{filename}", origin=0, writeable=False) 

130 self.assertEqual(roFromUri.filename, filename) 

131 self.assertEqual(roFromUri.origin, 0) 

132 self.assertFalse(roFromUri.isWriteable()) 

133 self.assertFalse(isEmptyDatabaseActuallyWriteable(roFromUri)) 

134 

135 

136class SqliteMemoryDatabaseTestCase(unittest.TestCase, DatabaseTests): 

137 """Tests for `SqliteDatabase` using an in-memory database. 

138 """ 

139 

140 def makeEmptyDatabase(self, origin: int = 0) -> SqliteDatabase: 

141 connection = SqliteDatabase.connect(filename=None) 

142 return SqliteDatabase.fromConnection(connection=connection, origin=origin) 

143 

144 def getNewConnection(self, database: SqliteDatabase, *, writeable: bool) -> SqliteDatabase: 

145 return SqliteDatabase.fromConnection(origin=database.origin, connection=database._connection, 

146 writeable=writeable) 

147 

148 @contextmanager 

149 def asReadOnly(self, database: SqliteDatabase) -> SqliteDatabase: 

150 yield self.getNewConnection(database, writeable=False) 

151 

152 def testConnection(self): 

153 """Test that different ways of connecting to a SQLite database 

154 are equivalent. 

155 """ 

156 # Create an in-memory database by passing filename=None. 

157 memFromFilename = SqliteDatabase.fromConnection(SqliteDatabase.connect(filename=None), origin=0) 

158 self.assertIsNone(memFromFilename.filename) 

159 self.assertEqual(memFromFilename.origin, 0) 

160 self.assertTrue(memFromFilename.isWriteable()) 

161 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromFilename)) 

162 # Create an in-memory database via a URI. 

163 memFromUri = SqliteDatabase.fromUri("sqlite://", origin=0) 

164 self.assertIsNone(memFromUri.filename) 

165 self.assertEqual(memFromUri.origin, 0) 

166 self.assertTrue(memFromUri.isWriteable()) 

167 self.assertTrue(isEmptyDatabaseActuallyWriteable(memFromUri)) 

168 # We don't support SQLite URIs inside SQLAlchemy URIs. 

169 with self.assertRaises(NotImplementedError): 

170 SqliteDatabase.connect(uri="sqlite:///:memory:?uri=true") 

171 # We don't support read-only in-memory databases. 

172 with self.assertRaises(NotImplementedError): 

173 SqliteDatabase.connect(filename=None, writeable=False) 

174 

175 

176class SqliteFileRegistryTestCase(unittest.TestCase, RegistryTests): 

177 """Tests for `Registry` backed by a SQLite file-based database. 

178 """ 

179 

180 def setUp(self): 

181 self.root = tempfile.mkdtemp(dir=TESTDIR) 

182 

183 def tearDown(self): 

184 if self.root is not None and os.path.exists(self.root): 

185 shutil.rmtree(self.root, ignore_errors=True) 

186 

187 @classmethod 

188 def getDataDir(cls) -> str: 

189 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

190 

191 def makeRegistry(self) -> Registry: 

192 _, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3") 

193 config = RegistryConfig() 

194 config["db"] = f"sqlite:///{filename}" 

195 return Registry.fromConfig(config, create=True, butlerRoot=self.root) 

196 

197 

198class SqliteMemoryRegistryTestCase(unittest.TestCase, RegistryTests): 

199 """Tests for `Registry` backed by a SQLite in-memory database. 

200 """ 

201 

202 @classmethod 

203 def getDataDir(cls) -> str: 

204 return os.path.normpath(os.path.join(os.path.dirname(__file__), "data", "registry")) 

205 

206 def makeRegistry(self) -> Registry: 

207 config = RegistryConfig() 

208 config["db"] = f"sqlite://" 

209 return Registry.fromConfig(config, create=True) 

210 

211 def testRegions(self): 

212 """Tests for using region fields in `Registry` dimensions. 

213 """ 

214 # TODO: the test regions used here are enormous (significant fractions 

215 # of the sphere), and that makes this test prohibitively slow on 

216 # most real databases. These should be made more realistic, and the 

217 # test moved to daf/butler/registry/tests/registry.py. 

218 registry = self.makeRegistry() 

219 regionTract = ConvexPolygon((UnitVector3d(1, 0, 0), 

220 UnitVector3d(0, 1, 0), 

221 UnitVector3d(0, 0, 1))) 

222 regionPatch = ConvexPolygon((UnitVector3d(1, 1, 0), 

223 UnitVector3d(0, 1, 0), 

224 UnitVector3d(0, 0, 1))) 

225 regionVisit = ConvexPolygon((UnitVector3d(1, 0, 0), 

226 UnitVector3d(0, 1, 1), 

227 UnitVector3d(0, 0, 1))) 

228 regionVisitDetector = ConvexPolygon((UnitVector3d(1, 0, 0), 

229 UnitVector3d(0, 1, 0), 

230 UnitVector3d(0, 1, 1))) 

231 for a, b in itertools.combinations((regionTract, regionPatch, regionVisit, regionVisitDetector), 2): 

232 self.assertNotEqual(a, b) 

233 

234 # This depends on current dimensions.yaml definitions 

235 self.assertEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0) 

236 

237 # Add some dimension entries 

238 registry.insertDimensionData("instrument", {"name": "DummyCam"}) 

239 registry.insertDimensionData("physical_filter", 

240 {"instrument": "DummyCam", "name": "dummy_r", "abstract_filter": "r"}, 

241 {"instrument": "DummyCam", "name": "dummy_i", "abstract_filter": "i"}) 

242 for detector in (1, 2, 3, 4, 5): 

243 registry.insertDimensionData("detector", {"instrument": "DummyCam", "id": detector, 

244 "full_name": str(detector)}) 

245 registry.insertDimensionData("visit", 

246 {"instrument": "DummyCam", "id": 0, "name": "zero", 

247 "physical_filter": "dummy_r", "region": regionVisit}, 

248 {"instrument": "DummyCam", "id": 1, "name": "one", 

249 "physical_filter": "dummy_i"}) 

250 registry.insertDimensionData("skymap", {"skymap": "DummySkyMap", "hash": bytes()}) 

251 registry.insertDimensionData("tract", {"skymap": "DummySkyMap", "tract": 0, "region": regionTract}) 

252 registry.insertDimensionData("patch", 

253 {"skymap": "DummySkyMap", "tract": 0, "patch": 0, 

254 "cell_x": 0, "cell_y": 0, "region": regionPatch}) 

255 registry.insertDimensionData("visit_detector_region", 

256 {"instrument": "DummyCam", "visit": 0, "detector": 2, 

257 "region": regionVisitDetector}) 

258 

259 def getRegion(dataId): 

260 return registry.expandDataId(dataId).region 

261 

262 # Get region for a tract 

263 self.assertEqual(regionTract, getRegion({"skymap": "DummySkyMap", "tract": 0})) 

264 # Attempt to get region for a non-existent tract 

265 with self.assertRaises(LookupError): 

266 getRegion({"skymap": "DummySkyMap", "tract": 1}) 

267 # Get region for a (tract, patch) combination 

268 self.assertEqual(regionPatch, getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 0})) 

269 # Get region for a non-existent (tract, patch) combination 

270 with self.assertRaises(LookupError): 

271 getRegion({"skymap": "DummySkyMap", "tract": 0, "patch": 1}) 

272 # Get region for a visit 

273 self.assertEqual(regionVisit, getRegion({"instrument": "DummyCam", "visit": 0})) 

274 # Attempt to get region for a non-existent visit 

275 with self.assertRaises(LookupError): 

276 getRegion({"instrument": "DummyCam", "visit": 10}) 

277 # Get region for a (visit, detector) combination 

278 self.assertEqual(regionVisitDetector, 

279 getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2})) 

280 # Attempt to get region for a non-existent (visit, detector) 

281 # combination. This returns None rather than raising because we don't 

282 # want to require the region record to be present. 

283 self.assertIsNone(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 3})) 

284 # getRegion for a dataId containing no spatial dimensions should 

285 # return None 

286 self.assertIsNone(getRegion({"instrument": "DummyCam"})) 

287 # getRegion for a mix of spatial dimensions should return 

288 # NotImplemented, at least until we get it implemented. 

289 self.assertIs(getRegion({"instrument": "DummyCam", "visit": 0, "detector": 2, 

290 "skymap": "DummySkyMap", "tract": 0}), 

291 NotImplemented) 

292 # Check if we can get the region for a skypix 

293 self.assertIsInstance(getRegion({"htm9": 1000}), ConvexPolygon) 

294 # patch_htm7_overlap should not be empty 

295 self.assertNotEqual(len(list(registry.queryDimensions(["patch", "htm7"]))), 0) 

296 

297 

298if __name__ == "__main__": 298 ↛ 299line 298 didn't jump to line 299, because the condition on line 298 was never true

299 unittest.main()