Coverage for tests/test_apdbSql.py: 48%

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for Apdb class.""" 

23 

24import gc 

25import os 

26import shutil 

27import tempfile 

28import unittest 

29from typing import Any 

30from unittest.mock import patch 

31 

32import sqlalchemy 

33 

34import lsst.utils.tests 

35from lsst.dax.apdb import Apdb, ApdbConfig, ApdbReplica, ApdbTables 

36from lsst.dax.apdb.pixelization import Pixelization 

37from lsst.dax.apdb.sql import ApdbSql, ApdbSqlConfig 

38from lsst.dax.apdb.tests import ApdbSchemaUpdateTest, ApdbTest 

39 

40try: 

41 import testing.postgresql 

42except ImportError: 

43 testing = None 

44 

45TEST_SCHEMA = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema.yaml") 

46# Schema that uses `datetime` for timestamps. 

47TEST_SCHEMA_DT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-datetime.yaml") 

48 

49 

50class ApdbSQLTest(ApdbTest): 

51 """A common base class for SQL APDB tests.""" 

52 

53 dia_object_index: str 

54 

55 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

56 """Create database and return its config.""" 

57 kw = { 

58 "schema_file": TEST_SCHEMA, 

59 "dia_object_index": self.dia_object_index, 

60 "enable_replica": self.enable_replica, 

61 } 

62 kw.update(kwargs) 

63 return ApdbSql.init_database(**kw) # type: ignore[arg-type] 

64 

65 def getDiaObjects_table(self) -> ApdbTables: 

66 """Return type of table returned from getDiaObjects method.""" 

67 return ApdbTables.DiaObject 

68 

69 def pixelization(self, config: ApdbConfig) -> Pixelization: 

70 """Return pixelization used by implementation.""" 

71 assert isinstance(config, ApdbSqlConfig), "Only expect ApdbSqlConfig here" 

72 return Pixelization("htm", config.pixelization.htm_level, config.pixelization.htm_max_ranges) 

73 

74 def test_connection_timeout(self) -> None: 

75 """Test that setting connection timeout does not break things.""" 

76 config = self.make_instance() 

77 assert isinstance(config, ApdbSqlConfig), "Only expect ApdbSqlConfig here" 

78 config.connection_config.connection_timeout = 60.0 

79 Apdb.from_config(config) 

80 

81 

82class ApdbSQLiteTestCase(ApdbSQLTest, unittest.TestCase): 

83 """A test case for ApdbSql class using SQLite backend.""" 

84 

85 fsrc_requires_id_list = True 

86 dia_object_index = "baseline" 

87 use_mjd = True 

88 

89 def setUp(self) -> None: 

90 self.tempdir = tempfile.mkdtemp() 

91 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

92 

93 def tearDown(self) -> None: 

94 shutil.rmtree(self.tempdir, ignore_errors=True) 

95 

96 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

97 return super().make_instance(db_url=self.db_url, **kwargs) 

98 

99 

100class ApdbSQLiteDatetimeTestCase(ApdbSQLTest, unittest.TestCase): 

101 """A test case for ApdbSql class using SQLite backend and schema with 

102 timestamps in TAI MJD. 

103 """ 

104 

105 fsrc_requires_id_list = True 

106 dia_object_index = "baseline" 

107 use_mjd = False 

108 

109 def setUp(self) -> None: 

110 self.tempdir = tempfile.mkdtemp() 

111 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

112 

113 def tearDown(self) -> None: 

114 shutil.rmtree(self.tempdir, ignore_errors=True) 

115 

116 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

117 if "schema_file" in kwargs: 

118 return super().make_instance(db_url=self.db_url, **kwargs) 

119 else: 

120 return super().make_instance(db_url=self.db_url, schema_file=TEST_SCHEMA_DT, **kwargs) 

121 

122 

123class ApdbSQLiteTestCaseLastObject(ApdbSQLiteTestCase): 

124 """A test case for ApdbSql class using SQLite backend and DiaObjectLast 

125 table. 

126 """ 

127 

128 dia_object_index = "last_object_table" 

129 

130 def getDiaObjects_table(self) -> ApdbTables: 

131 """Return type of table returned from getDiaObjects method.""" 

132 return ApdbTables.DiaObjectLast 

133 

134 

135class ApdbSQLiteTestCasePixIdIovIndex(ApdbSQLiteTestCase): 

136 """A test case for ApdbSql class using SQLite backend with pix_id_iov 

137 indexing. 

138 """ 

139 

140 dia_object_index = "pix_id_iov" 

141 

142 

143class ApdbSQLiteTestCaseReplica(ApdbSQLiteTestCase): 

144 """Test case for ApdbSql class using SQLite backend with replica tables.""" 

145 

146 enable_replica = True 

147 meta_row_count = 4 

148 

149 

150@unittest.skipUnless(testing is not None, "testing.postgresql module not found") 

151class ApdbPostgresTestCase(ApdbSQLTest, unittest.TestCase): 

152 """A test case for ApdbSql class using Postgres backend.""" 

153 

154 fsrc_requires_id_list = True 

155 dia_object_index = "last_object_table" 

156 postgresql: Any 

157 enable_replica = True 

158 meta_row_count = 4 

159 timestamp_type_name = "datetime64[ns]" 

160 

161 @classmethod 

162 def setUpClass(cls) -> None: 

163 # Create the postgres test server. 

164 cls.postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) 

165 super().setUpClass() 

166 

167 @classmethod 

168 def tearDownClass(cls) -> None: 

169 # Clean up any lingering SQLAlchemy engines/connections 

170 # so they're closed before we shut down the server. 

171 gc.collect() 

172 cls.postgresql.clear_cache() 

173 super().tearDownClass() 

174 

175 def setUp(self) -> None: 

176 self.server = self.postgresql() 

177 

178 def tearDown(self) -> None: 

179 self.server = self.postgresql() 

180 

181 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

182 return super().make_instance(db_url=self.server.url(), **kwargs) 

183 

184 def getDiaObjects_table(self) -> ApdbTables: 

185 """Return type of table returned from getDiaObjects method.""" 

186 return ApdbTables.DiaObjectLast 

187 

188 

189@unittest.skipUnless(testing is not None, "testing.postgresql module not found") 

190class ApdbPostgresNamespaceTestCase(ApdbPostgresTestCase): 

191 """A test case for ApdbSql class using Postgres backend with schema name""" 

192 

193 # use mixed case to trigger quoting 

194 namespace = "ApdbSchema" 

195 

196 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

197 """Make config class instance used in all tests.""" 

198 return super().make_instance(namespace=self.namespace, **kwargs) 

199 

200 

201class ApdbSchemaUpdateSQLiteTestCase(ApdbSchemaUpdateTest, unittest.TestCase): 

202 """A test case for schema updates using SQLite backend.""" 

203 

204 def setUp(self) -> None: 

205 self.tempdir = tempfile.mkdtemp() 

206 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

207 

208 def tearDown(self) -> None: 

209 shutil.rmtree(self.tempdir, ignore_errors=True) 

210 

211 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

212 """Make config class instance used in all tests.""" 

213 kw = { 

214 "db_url": self.db_url, 

215 "schema_file": TEST_SCHEMA, 

216 } 

217 kw.update(kwargs) 

218 return ApdbSql.init_database(**kw) # type: ignore[arg-type] 

219 

220 

221class ApdbSQLiteFromUriTestCase(unittest.TestCase): 

222 """A test case for for instantiating ApdbSql via URI.""" 

223 

224 def setUp(self) -> None: 

225 self.tempdir = tempfile.mkdtemp() 

226 self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) 

227 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

228 config = ApdbSql.init_database(db_url=self.db_url, schema_file=TEST_SCHEMA) 

229 # TODO: This will need update when we switch to pydantic configs. 

230 self.config_path = os.path.join(self.tempdir, "apdb-config.yaml") 

231 config.save(self.config_path) 

232 self.bad_config_path = os.path.join(self.tempdir, "not-config.yaml") 

233 self.index_path = os.path.join(self.tempdir, "apdb-index.yaml") 

234 with open(self.index_path, "w") as index_file: 

235 print(f'label1: "{self.config_path}"', file=index_file) 

236 print(f'"label2/pex_config": "{self.config_path}"', file=index_file) 

237 print(f'bad-label: "{self.bad_config_path}"', file=index_file) 

238 # File with incorrect format. 

239 self.bad_index_path = os.path.join(self.tempdir, "apdb-index-bad.yaml") 

240 with open(self.bad_index_path, "w") as index_file: 

241 print(f'label1: ["{self.config_path}"]', file=index_file) 

242 self.missing_index_path = os.path.join(self.tempdir, "no-apdb-index.yaml") 

243 

244 def test_make_apdb_from_path(self) -> None: 

245 """Check that we can make APDB instance from config URI.""" 

246 Apdb.from_uri(self.config_path) 

247 with self.assertRaises(FileNotFoundError): 

248 Apdb.from_uri(self.bad_config_path) 

249 

250 def test_make_apdb_from_labels(self) -> None: 

251 """Check that we can make APDB instance from config URI.""" 

252 # Replace DAX_APDB_INDEX_URI value 

253 new_env = {"DAX_APDB_INDEX_URI": self.index_path} 

254 with patch.dict(os.environ, new_env, clear=True): 

255 Apdb.from_uri("label:label1") 

256 Apdb.from_uri("label:label2") 

257 # Label does not exist. 

258 with self.assertRaises(ValueError): 

259 Apdb.from_uri("label:not-a-label") 

260 # Label exists but points to a missing config. 

261 with self.assertRaises(FileNotFoundError): 

262 Apdb.from_uri("label:bad-label") 

263 

264 def test_make_apdb_bad_index(self) -> None: 

265 """Check what happens when DAX_APDB_INDEX_URI is broken.""" 

266 # envvar is set but empty. 

267 new_env = {"DAX_APDB_INDEX_URI": ""} 

268 with patch.dict(os.environ, new_env, clear=True): 

269 with self.assertRaises(RuntimeError): 

270 Apdb.from_uri("label:label") 

271 

272 # envvar is set to something non-existing. 

273 new_env = {"DAX_APDB_INDEX_URI": self.missing_index_path} 

274 with patch.dict(os.environ, new_env, clear=True): 

275 with self.assertRaises(FileNotFoundError): 

276 Apdb.from_uri("label:label") 

277 

278 # envvar points to an incorrect file. 

279 new_env = {"DAX_APDB_INDEX_URI": self.bad_index_path} 

280 with patch.dict(os.environ, new_env, clear=True): 

281 with self.assertRaises(TypeError): 

282 Apdb.from_uri("label:label") 

283 

284 def test_remove_database_file(self) -> None: 

285 """Check that SQLite does not try to recreate database after it was 

286 removed, but config persisted. 

287 """ 

288 os.unlink(f"{self.tempdir}/apdb.sqlite3") 

289 with self.assertRaisesRegex(sqlalchemy.exc.OperationalError, "unable to open database file"): 

290 Apdb.from_uri(self.config_path) 

291 

292 def test_make_apdb_replica(self) -> None: 

293 """Check that we can make ApdbReplica instance from config URI.""" 

294 ApdbReplica.from_uri(self.config_path) 

295 with self.assertRaises(FileNotFoundError): 

296 Apdb.from_uri(self.bad_config_path) 

297 

298 

299class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

300 """Run file leak tests.""" 

301 

302 

303def setup_module(module: Any) -> None: 

304 """Configure pytest.""" 

305 lsst.utils.tests.init() 

306 

307 

308if __name__ == "__main__": 

309 lsst.utils.tests.init() 

310 unittest.main()