Coverage for tests / test_apdbSql.py: 47%

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for Apdb class.""" 

23 

24import gc 

25import os 

26import shutil 

27import tempfile 

28import unittest 

29from typing import Any 

30from unittest.mock import patch 

31 

32import sqlalchemy 

33 

34import lsst.utils.tests 

35from lsst.dax.apdb import ( 

36 Apdb, 

37 ApdbConfig, 

38 ApdbReplica, 

39 ApdbTables, 

40 ApdbUpdateRecord, 

41 IncompatibleVersionError, 

42 ReplicaChunk, 

43) 

44from lsst.dax.apdb.pixelization import Pixelization 

45from lsst.dax.apdb.sql import ApdbSql, ApdbSqlConfig 

46from lsst.dax.apdb.tests import ApdbSchemaUpdateTest, ApdbTest 

47 

48try: 

49 import testing.postgresql 

50except ImportError: 

51 testing = None 

52 

53TEST_SCHEMA = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-apdb.yaml") 

54TEST_SCHEMA_SSO = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-sso.yaml") 

55# Schema that uses `datetime` for timestamps and combines APDB and SSP. 

56TEST_SCHEMA_DT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-datetime.yaml") 

57 

58 

59class ApdbSQLTest(ApdbTest): 

60 """A common base class for SQL APDB tests.""" 

61 

62 dia_object_index: str 

63 

64 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

65 """Create database and return its config.""" 

66 kw = { 

67 "schema_file": TEST_SCHEMA, 

68 "ss_schema_file": TEST_SCHEMA_SSO, 

69 "dia_object_index": self.dia_object_index, 

70 "enable_replica": self.enable_replica, 

71 } 

72 kw.update(kwargs) 

73 return ApdbSql.init_database(**kw) # type: ignore[arg-type] 

74 

75 def getDiaObjects_table(self) -> ApdbTables: 

76 """Return type of table returned from getDiaObjects method.""" 

77 return ApdbTables.DiaObject 

78 

79 def pixelization(self, config: ApdbConfig) -> Pixelization: 

80 """Return pixelization used by implementation.""" 

81 assert isinstance(config, ApdbSqlConfig), "Only expect ApdbSqlConfig here" 

82 return Pixelization("htm", config.pixelization.htm_level, config.pixelization.htm_max_ranges) 

83 

84 def test_connection_timeout(self) -> None: 

85 """Test that setting connection timeout does not break things.""" 

86 config = self.make_instance() 

87 assert isinstance(config, ApdbSqlConfig), "Only expect ApdbSqlConfig here" 

88 config.connection_config.connection_timeout = 60.0 

89 Apdb.from_config(config) 

90 

91 def store_update_records(self, apdb: Apdb, records: list[ApdbUpdateRecord], chunk: ReplicaChunk) -> None: 

92 # Docstring inherited. 

93 assert isinstance(apdb, ApdbSql), "Expecting ApdbSql instance" 

94 apdb._storeUpdateRecords(records, chunk, store_chunk=True) 

95 

96 

97class ApdbSQLiteTestCase(ApdbSQLTest, unittest.TestCase): 

98 """A test case for ApdbSql class using SQLite backend.""" 

99 

100 fsrc_requires_id_list = True 

101 dia_object_index = "baseline" 

102 use_mjd = True 

103 

104 def setUp(self) -> None: 

105 self.tempdir = tempfile.mkdtemp() 

106 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

107 

108 def tearDown(self) -> None: 

109 shutil.rmtree(self.tempdir, ignore_errors=True) 

110 

111 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

112 return super().make_instance(db_url=self.db_url, **kwargs) 

113 

114 

115class ApdbSQLiteDatetimeTestCase(ApdbSQLTest, unittest.TestCase): 

116 """A test case for ApdbSql class using SQLite backend and schema with 

117 timestamps in TAI MJD. 

118 """ 

119 

120 fsrc_requires_id_list = True 

121 dia_object_index = "baseline" 

122 use_mjd = False 

123 

124 def setUp(self) -> None: 

125 self.tempdir = tempfile.mkdtemp() 

126 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

127 

128 def tearDown(self) -> None: 

129 shutil.rmtree(self.tempdir, ignore_errors=True) 

130 

131 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

132 if "schema_file" in kwargs: 

133 return super().make_instance(db_url=self.db_url, **kwargs) 

134 else: 

135 return super().make_instance(db_url=self.db_url, schema_file=TEST_SCHEMA_DT, **kwargs) 

136 

137 

138class ApdbSQLiteTestCaseLastObject(ApdbSQLiteTestCase): 

139 """A test case for ApdbSql class using SQLite backend and DiaObjectLast 

140 table. 

141 """ 

142 

143 dia_object_index = "last_object_table" 

144 

145 def getDiaObjects_table(self) -> ApdbTables: 

146 """Return type of table returned from getDiaObjects method.""" 

147 return ApdbTables.DiaObjectLast 

148 

149 

150class ApdbSQLiteTestCasePixIdIovIndex(ApdbSQLiteTestCase): 

151 """A test case for ApdbSql class using SQLite backend with pix_id_iov 

152 indexing. 

153 """ 

154 

155 dia_object_index = "pix_id_iov" 

156 

157 

158class ApdbSQLiteTestCaseReplica(ApdbSQLiteTestCase): 

159 """Test case for ApdbSql class using SQLite backend with replica tables.""" 

160 

161 enable_replica = True 

162 meta_row_count = 4 

163 

164 

165@unittest.skipUnless(testing is not None, "testing.postgresql module not found") 

166class ApdbPostgresTestCase(ApdbSQLTest, unittest.TestCase): 

167 """A test case for ApdbSql class using Postgres backend.""" 

168 

169 fsrc_requires_id_list = True 

170 dia_object_index = "last_object_table" 

171 postgresql: Any 

172 enable_replica = True 

173 meta_row_count = 4 

174 timestamp_type_name = "datetime64[ns]" 

175 

176 @classmethod 

177 def setUpClass(cls) -> None: 

178 # Create the postgres test server. 

179 cls.postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) 

180 super().setUpClass() 

181 

182 @classmethod 

183 def tearDownClass(cls) -> None: 

184 # Clean up any lingering SQLAlchemy engines/connections 

185 # so they're closed before we shut down the server. 

186 gc.collect() 

187 cls.postgresql.clear_cache() 

188 super().tearDownClass() 

189 

190 def setUp(self) -> None: 

191 self.server = self.postgresql() 

192 

193 def tearDown(self) -> None: 

194 self.server = self.postgresql() 

195 

196 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

197 return super().make_instance(db_url=self.server.url(), **kwargs) 

198 

199 def getDiaObjects_table(self) -> ApdbTables: 

200 """Return type of table returned from getDiaObjects method.""" 

201 return ApdbTables.DiaObjectLast 

202 

203 

204@unittest.skipUnless(testing is not None, "testing.postgresql module not found") 

205class ApdbPostgresNamespaceTestCase(ApdbPostgresTestCase): 

206 """A test case for ApdbSql class using Postgres backend with schema name""" 

207 

208 # use mixed case to trigger quoting 

209 namespace = "ApdbSchema" 

210 

211 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

212 """Make config class instance used in all tests.""" 

213 return super().make_instance(namespace=self.namespace, **kwargs) 

214 

215 

216class ApdbSchemaUpdateSQLiteTestCase(ApdbSchemaUpdateTest, unittest.TestCase): 

217 """A test case for schema updates using SQLite backend.""" 

218 

219 def setUp(self) -> None: 

220 self.tempdir = tempfile.mkdtemp() 

221 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

222 

223 def tearDown(self) -> None: 

224 shutil.rmtree(self.tempdir, ignore_errors=True) 

225 

226 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

227 """Make config class instance used in all tests.""" 

228 kw = { 

229 "db_url": self.db_url, 

230 "schema_file": TEST_SCHEMA, 

231 "ss_schema_file": TEST_SCHEMA_SSO, 

232 } 

233 kw.update(kwargs) 

234 return ApdbSql.init_database(**kw) # type: ignore[arg-type] 

235 

236 

237class ApdbSQLiteFromUriTestCase(unittest.TestCase): 

238 """A test case for instantiating ApdbSql via URI.""" 

239 

240 def setUp(self) -> None: 

241 self.tempdir = tempfile.mkdtemp() 

242 self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) 

243 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

244 config = ApdbSql.init_database( 

245 db_url=self.db_url, schema_file=TEST_SCHEMA, ss_schema_file=TEST_SCHEMA_SSO 

246 ) 

247 # TODO: This will need update when we switch to pydantic configs. 

248 self.config_path = os.path.join(self.tempdir, "apdb-config.yaml") 

249 config.save(self.config_path) 

250 self.bad_config_path = os.path.join(self.tempdir, "not-config.yaml") 

251 self.index_path = os.path.join(self.tempdir, "apdb-index.yaml") 

252 with open(self.index_path, "w") as index_file: 

253 print(f'label1: "{self.config_path}"', file=index_file) 

254 print(f'"label2/pex_config": "{self.config_path}"', file=index_file) 

255 print(f'bad-label: "{self.bad_config_path}"', file=index_file) 

256 # File with incorrect format. 

257 self.bad_index_path = os.path.join(self.tempdir, "apdb-index-bad.yaml") 

258 with open(self.bad_index_path, "w") as index_file: 

259 print(f'label1: ["{self.config_path}"]', file=index_file) 

260 self.missing_index_path = os.path.join(self.tempdir, "no-apdb-index.yaml") 

261 

262 def test_make_apdb_from_path(self) -> None: 

263 """Check that we can make APDB instance from config URI.""" 

264 Apdb.from_uri(self.config_path) 

265 with self.assertRaises(FileNotFoundError): 

266 Apdb.from_uri(self.bad_config_path) 

267 

268 def test_make_apdb_from_labels(self) -> None: 

269 """Check that we can make APDB instance from config URI.""" 

270 # Replace DAX_APDB_INDEX_URI value 

271 new_env = {"DAX_APDB_INDEX_URI": self.index_path} 

272 with patch.dict(os.environ, new_env, clear=True): 

273 Apdb.from_uri("label:label1") 

274 Apdb.from_uri("label:label2") 

275 # Label does not exist. 

276 with self.assertRaises(ValueError): 

277 Apdb.from_uri("label:not-a-label") 

278 # Label exists but points to a missing config. 

279 with self.assertRaises(FileNotFoundError): 

280 Apdb.from_uri("label:bad-label") 

281 

282 def test_make_apdb_bad_index(self) -> None: 

283 """Check what happens when DAX_APDB_INDEX_URI is broken.""" 

284 # envvar is set but empty. 

285 new_env = {"DAX_APDB_INDEX_URI": ""} 

286 with patch.dict(os.environ, new_env, clear=True): 

287 with self.assertRaises(RuntimeError): 

288 Apdb.from_uri("label:label") 

289 

290 # envvar is set to something non-existing. 

291 new_env = {"DAX_APDB_INDEX_URI": self.missing_index_path} 

292 with patch.dict(os.environ, new_env, clear=True): 

293 with self.assertRaises(FileNotFoundError): 

294 Apdb.from_uri("label:label") 

295 

296 # envvar points to an incorrect file. 

297 new_env = {"DAX_APDB_INDEX_URI": self.bad_index_path} 

298 with patch.dict(os.environ, new_env, clear=True): 

299 with self.assertRaises(TypeError): 

300 Apdb.from_uri("label:label") 

301 

302 def test_remove_database_file(self) -> None: 

303 """Check that SQLite does not try to recreate database after it was 

304 removed, but config persisted. 

305 """ 

306 os.unlink(f"{self.tempdir}/apdb.sqlite3") 

307 with self.assertRaisesRegex(sqlalchemy.exc.OperationalError, "unable to open database file"): 

308 Apdb.from_uri(self.config_path) 

309 

310 def test_make_apdb_replica(self) -> None: 

311 """Check that we can make ApdbReplica instance from config URI.""" 

312 ApdbReplica.from_uri(self.config_path) 

313 with self.assertRaises(FileNotFoundError): 

314 Apdb.from_uri(self.bad_config_path) 

315 

316 

317class ApdbSQLiteVersionCheck(unittest.TestCase): 

318 """A test case to verify that version check happens before reading 

319 frozen configuration. 

320 """ 

321 

322 def setUp(self) -> None: 

323 self.tempdir = tempfile.mkdtemp() 

324 self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) 

325 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

326 self.config = ApdbSql.init_database( 

327 db_url=self.db_url, schema_file=TEST_SCHEMA, ss_schema_file=TEST_SCHEMA_SSO 

328 ) 

329 

330 def test_version_check(self) -> None: 

331 """Test that version check happens before reading config.""" 

332 apdb = Apdb.from_config(self.config) 

333 assert isinstance(apdb, ApdbSql) 

334 

335 # Store incompatible version. 

336 apdb.metadata.set(ApdbSql.metadataSchemaVersionKey, "99.0.0", force=True) 

337 

338 # Overwrite frozen config with something that will break. 

339 apdb.metadata.set(ApdbSql.metadataConfigKey, '{"not_a_config_key": 0}', force=True) 

340 

341 # Try again. 

342 with self.assertRaises(IncompatibleVersionError): 

343 Apdb.from_config(self.config) 

344 

345 

346class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

347 """Run file leak tests.""" 

348 

349 

350def setup_module(module: Any) -> None: 

351 """Configure pytest.""" 

352 lsst.utils.tests.init() 

353 

354 

355if __name__ == "__main__": 

356 lsst.utils.tests.init() 

357 unittest.main()