Coverage for tests / test_apdbSql.py: 49%

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for Apdb class.""" 

23 

24import gc 

25import os 

26import shutil 

27import tempfile 

28import unittest 

29from typing import Any 

30from unittest.mock import patch 

31 

32import sqlalchemy 

33 

34import lsst.utils.tests 

35from lsst.dax.apdb import Apdb, ApdbConfig, ApdbReplica, ApdbTables, ApdbUpdateRecord, ReplicaChunk 

36from lsst.dax.apdb.pixelization import Pixelization 

37from lsst.dax.apdb.sql import ApdbSql, ApdbSqlConfig 

38from lsst.dax.apdb.tests import ApdbSchemaUpdateTest, ApdbTest 

39 

40try: 

41 import testing.postgresql 

42except ImportError: 

43 testing = None 

44 

45TEST_SCHEMA = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-apdb.yaml") 

46TEST_SCHEMA_SSO = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-sso.yaml") 

47# Schema that uses `datetime` for timestamps and combines APDB and SSP. 

48TEST_SCHEMA_DT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-datetime.yaml") 

49 

50 

51class ApdbSQLTest(ApdbTest): 

52 """A common base class for SQL APDB tests.""" 

53 

54 dia_object_index: str 

55 

56 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

57 """Create database and return its config.""" 

58 kw = { 

59 "schema_file": TEST_SCHEMA, 

60 "ss_schema_file": TEST_SCHEMA_SSO, 

61 "dia_object_index": self.dia_object_index, 

62 "enable_replica": self.enable_replica, 

63 } 

64 kw.update(kwargs) 

65 return ApdbSql.init_database(**kw) # type: ignore[arg-type] 

66 

67 def getDiaObjects_table(self) -> ApdbTables: 

68 """Return type of table returned from getDiaObjects method.""" 

69 return ApdbTables.DiaObject 

70 

71 def pixelization(self, config: ApdbConfig) -> Pixelization: 

72 """Return pixelization used by implementation.""" 

73 assert isinstance(config, ApdbSqlConfig), "Only expect ApdbSqlConfig here" 

74 return Pixelization("htm", config.pixelization.htm_level, config.pixelization.htm_max_ranges) 

75 

76 def test_connection_timeout(self) -> None: 

77 """Test that setting connection timeout does not break things.""" 

78 config = self.make_instance() 

79 assert isinstance(config, ApdbSqlConfig), "Only expect ApdbSqlConfig here" 

80 config.connection_config.connection_timeout = 60.0 

81 Apdb.from_config(config) 

82 

83 def store_update_records(self, apdb: Apdb, records: list[ApdbUpdateRecord], chunk: ReplicaChunk) -> None: 

84 # Docstring inherited. 

85 assert isinstance(apdb, ApdbSql), "Expecting ApdbSql instance" 

86 apdb._storeUpdateRecords(records, chunk, store_chunk=True) 

87 

88 

89class ApdbSQLiteTestCase(ApdbSQLTest, unittest.TestCase): 

90 """A test case for ApdbSql class using SQLite backend.""" 

91 

92 fsrc_requires_id_list = True 

93 dia_object_index = "baseline" 

94 use_mjd = True 

95 

96 def setUp(self) -> None: 

97 self.tempdir = tempfile.mkdtemp() 

98 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

99 

100 def tearDown(self) -> None: 

101 shutil.rmtree(self.tempdir, ignore_errors=True) 

102 

103 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

104 return super().make_instance(db_url=self.db_url, **kwargs) 

105 

106 

107class ApdbSQLiteDatetimeTestCase(ApdbSQLTest, unittest.TestCase): 

108 """A test case for ApdbSql class using SQLite backend and schema with 

109 timestamps in TAI MJD. 

110 """ 

111 

112 fsrc_requires_id_list = True 

113 dia_object_index = "baseline" 

114 use_mjd = False 

115 

116 def setUp(self) -> None: 

117 self.tempdir = tempfile.mkdtemp() 

118 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

119 

120 def tearDown(self) -> None: 

121 shutil.rmtree(self.tempdir, ignore_errors=True) 

122 

123 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

124 if "schema_file" in kwargs: 

125 return super().make_instance(db_url=self.db_url, **kwargs) 

126 else: 

127 return super().make_instance(db_url=self.db_url, schema_file=TEST_SCHEMA_DT, **kwargs) 

128 

129 

130class ApdbSQLiteTestCaseLastObject(ApdbSQLiteTestCase): 

131 """A test case for ApdbSql class using SQLite backend and DiaObjectLast 

132 table. 

133 """ 

134 

135 dia_object_index = "last_object_table" 

136 

137 def getDiaObjects_table(self) -> ApdbTables: 

138 """Return type of table returned from getDiaObjects method.""" 

139 return ApdbTables.DiaObjectLast 

140 

141 

142class ApdbSQLiteTestCasePixIdIovIndex(ApdbSQLiteTestCase): 

143 """A test case for ApdbSql class using SQLite backend with pix_id_iov 

144 indexing. 

145 """ 

146 

147 dia_object_index = "pix_id_iov" 

148 

149 

150class ApdbSQLiteTestCaseReplica(ApdbSQLiteTestCase): 

151 """Test case for ApdbSql class using SQLite backend with replica tables.""" 

152 

153 enable_replica = True 

154 meta_row_count = 4 

155 

156 

157@unittest.skipUnless(testing is not None, "testing.postgresql module not found") 

158class ApdbPostgresTestCase(ApdbSQLTest, unittest.TestCase): 

159 """A test case for ApdbSql class using Postgres backend.""" 

160 

161 fsrc_requires_id_list = True 

162 dia_object_index = "last_object_table" 

163 postgresql: Any 

164 enable_replica = True 

165 meta_row_count = 4 

166 timestamp_type_name = "datetime64[ns]" 

167 

168 @classmethod 

169 def setUpClass(cls) -> None: 

170 # Create the postgres test server. 

171 cls.postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) 

172 super().setUpClass() 

173 

174 @classmethod 

175 def tearDownClass(cls) -> None: 

176 # Clean up any lingering SQLAlchemy engines/connections 

177 # so they're closed before we shut down the server. 

178 gc.collect() 

179 cls.postgresql.clear_cache() 

180 super().tearDownClass() 

181 

182 def setUp(self) -> None: 

183 self.server = self.postgresql() 

184 

185 def tearDown(self) -> None: 

186 self.server = self.postgresql() 

187 

188 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

189 return super().make_instance(db_url=self.server.url(), **kwargs) 

190 

191 def getDiaObjects_table(self) -> ApdbTables: 

192 """Return type of table returned from getDiaObjects method.""" 

193 return ApdbTables.DiaObjectLast 

194 

195 

196@unittest.skipUnless(testing is not None, "testing.postgresql module not found") 

197class ApdbPostgresNamespaceTestCase(ApdbPostgresTestCase): 

198 """A test case for ApdbSql class using Postgres backend with schema name""" 

199 

200 # use mixed case to trigger quoting 

201 namespace = "ApdbSchema" 

202 

203 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

204 """Make config class instance used in all tests.""" 

205 return super().make_instance(namespace=self.namespace, **kwargs) 

206 

207 

208class ApdbSchemaUpdateSQLiteTestCase(ApdbSchemaUpdateTest, unittest.TestCase): 

209 """A test case for schema updates using SQLite backend.""" 

210 

211 def setUp(self) -> None: 

212 self.tempdir = tempfile.mkdtemp() 

213 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

214 

215 def tearDown(self) -> None: 

216 shutil.rmtree(self.tempdir, ignore_errors=True) 

217 

218 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

219 """Make config class instance used in all tests.""" 

220 kw = { 

221 "db_url": self.db_url, 

222 "schema_file": TEST_SCHEMA, 

223 "ss_schema_file": TEST_SCHEMA_SSO, 

224 } 

225 kw.update(kwargs) 

226 return ApdbSql.init_database(**kw) # type: ignore[arg-type] 

227 

228 

229class ApdbSQLiteFromUriTestCase(unittest.TestCase): 

230 """A test case for for instantiating ApdbSql via URI.""" 

231 

232 def setUp(self) -> None: 

233 self.tempdir = tempfile.mkdtemp() 

234 self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) 

235 self.db_url = f"sqlite:///{self.tempdir}/apdb.sqlite3" 

236 config = ApdbSql.init_database( 

237 db_url=self.db_url, schema_file=TEST_SCHEMA, ss_schema_file=TEST_SCHEMA_SSO 

238 ) 

239 # TODO: This will need update when we switch to pydantic configs. 

240 self.config_path = os.path.join(self.tempdir, "apdb-config.yaml") 

241 config.save(self.config_path) 

242 self.bad_config_path = os.path.join(self.tempdir, "not-config.yaml") 

243 self.index_path = os.path.join(self.tempdir, "apdb-index.yaml") 

244 with open(self.index_path, "w") as index_file: 

245 print(f'label1: "{self.config_path}"', file=index_file) 

246 print(f'"label2/pex_config": "{self.config_path}"', file=index_file) 

247 print(f'bad-label: "{self.bad_config_path}"', file=index_file) 

248 # File with incorrect format. 

249 self.bad_index_path = os.path.join(self.tempdir, "apdb-index-bad.yaml") 

250 with open(self.bad_index_path, "w") as index_file: 

251 print(f'label1: ["{self.config_path}"]', file=index_file) 

252 self.missing_index_path = os.path.join(self.tempdir, "no-apdb-index.yaml") 

253 

254 def test_make_apdb_from_path(self) -> None: 

255 """Check that we can make APDB instance from config URI.""" 

256 Apdb.from_uri(self.config_path) 

257 with self.assertRaises(FileNotFoundError): 

258 Apdb.from_uri(self.bad_config_path) 

259 

260 def test_make_apdb_from_labels(self) -> None: 

261 """Check that we can make APDB instance from config URI.""" 

262 # Replace DAX_APDB_INDEX_URI value 

263 new_env = {"DAX_APDB_INDEX_URI": self.index_path} 

264 with patch.dict(os.environ, new_env, clear=True): 

265 Apdb.from_uri("label:label1") 

266 Apdb.from_uri("label:label2") 

267 # Label does not exist. 

268 with self.assertRaises(ValueError): 

269 Apdb.from_uri("label:not-a-label") 

270 # Label exists but points to a missing config. 

271 with self.assertRaises(FileNotFoundError): 

272 Apdb.from_uri("label:bad-label") 

273 

274 def test_make_apdb_bad_index(self) -> None: 

275 """Check what happens when DAX_APDB_INDEX_URI is broken.""" 

276 # envvar is set but empty. 

277 new_env = {"DAX_APDB_INDEX_URI": ""} 

278 with patch.dict(os.environ, new_env, clear=True): 

279 with self.assertRaises(RuntimeError): 

280 Apdb.from_uri("label:label") 

281 

282 # envvar is set to something non-existing. 

283 new_env = {"DAX_APDB_INDEX_URI": self.missing_index_path} 

284 with patch.dict(os.environ, new_env, clear=True): 

285 with self.assertRaises(FileNotFoundError): 

286 Apdb.from_uri("label:label") 

287 

288 # envvar points to an incorrect file. 

289 new_env = {"DAX_APDB_INDEX_URI": self.bad_index_path} 

290 with patch.dict(os.environ, new_env, clear=True): 

291 with self.assertRaises(TypeError): 

292 Apdb.from_uri("label:label") 

293 

294 def test_remove_database_file(self) -> None: 

295 """Check that SQLite does not try to recreate database after it was 

296 removed, but config persisted. 

297 """ 

298 os.unlink(f"{self.tempdir}/apdb.sqlite3") 

299 with self.assertRaisesRegex(sqlalchemy.exc.OperationalError, "unable to open database file"): 

300 Apdb.from_uri(self.config_path) 

301 

302 def test_make_apdb_replica(self) -> None: 

303 """Check that we can make ApdbReplica instance from config URI.""" 

304 ApdbReplica.from_uri(self.config_path) 

305 with self.assertRaises(FileNotFoundError): 

306 Apdb.from_uri(self.bad_config_path) 

307 

308 

309class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

310 """Run file leak tests.""" 

311 

312 

313def setup_module(module: Any) -> None: 

314 """Configure pytest.""" 

315 lsst.utils.tests.init() 

316 

317 

318if __name__ == "__main__": 

319 lsst.utils.tests.init() 

320 unittest.main()