Coverage for tests / test_apdbCassandra.py: 49%

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for `ApdbCassandra` class. 

23 

24Notes 

25----- 

26For now this test can only run against actual Cassandra cluster, to specify 

27cluster location use ``DAX_APDB_TEST_CASSANDRA_CLUSTER`` environment variable, 

28e.g.: 

29 

30 export DAX_APDB_TEST_CASSANDRA_CLUSTER=cassandra.example.com 

31 pytest tests/test_apdbCassandra.py 

32 

33Individual tests create and destroy unique keyspaces in the cluster, there is 

34no need to pre-create a keyspace with predefined name. 

35""" 

36 

37import os 

38import unittest 

39from typing import Any 

40 

41import astropy.time 

42 

43import lsst.utils.tests 

44from lsst.dax.apdb import ( 

45 Apdb, 

46 ApdbConfig, 

47 ApdbTables, 

48 ApdbUpdateRecord, 

49 IncompatibleVersionError, 

50 ReplicaChunk, 

51) 

52from lsst.dax.apdb.cassandra import ApdbCassandra, ApdbCassandraConfig 

53from lsst.dax.apdb.cassandra.connectionContext import ConnectionContext 

54from lsst.dax.apdb.pixelization import Pixelization 

55from lsst.dax.apdb.tests import ApdbSchemaUpdateTest, ApdbTest, cassandra_mixin 

56from lsst.dax.apdb.tests.data_factory import makeObjectCatalog 

57 

58TEST_SCHEMA = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-apdb.yaml") 

59TEST_SCHEMA_SSO = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-sso.yaml") 

60# Schema that uses `datetime` for timestamps and combines APDB and SSP. 

61TEST_SCHEMA_DT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "config/schema-datetime.yaml") 

62 

63 

64class ApdbCassandraMixin(cassandra_mixin.ApdbCassandraMixin): 

65 """Mixin class which defines common methods for unit tests.""" 

66 

67 def pixelization(self, config: ApdbConfig) -> Pixelization: 

68 """Return pixelization used by implementation.""" 

69 assert isinstance(config, ApdbCassandraConfig), "Only expect ApdbCassandraConfig here" 

70 return Pixelization( 

71 config.partitioning.part_pixelization, 

72 config.partitioning.part_pix_level, 

73 config.partitioning.part_pix_max_ranges, 

74 ) 

75 

76 

77class ApdbCassandraTestCase(ApdbCassandraMixin, ApdbTest, unittest.TestCase): 

78 """A test case for ApdbCassandra class""" 

79 

80 time_partition_tables = False 

81 time_partition_start: str | None = None 

82 time_partition_end: str | None = None 

83 extra_chunk_columns = 2 

84 

85 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

86 """Make config class instance used in all tests.""" 

87 kw: dict[str, Any] = { 

88 "hosts": (self.cluster_host,), 

89 "keyspace": self.keyspace, 

90 "schema_file": TEST_SCHEMA, 

91 "ss_schema_file": TEST_SCHEMA_SSO, 

92 "time_partition_tables": self.time_partition_tables, 

93 "enable_replica": self.enable_replica, 

94 } 

95 if self.time_partition_start: 

96 kw["time_partition_start"] = self.time_partition_start 

97 if self.time_partition_end: 

98 kw["time_partition_end"] = self.time_partition_end 

99 kw.update(kwargs) 

100 return ApdbCassandra.init_database(**kw) 

101 

102 def getDiaObjects_table(self) -> ApdbTables: 

103 """Return type of table returned from getDiaObjects method.""" 

104 return ApdbTables.DiaObjectLast 

105 

106 def store_update_records(self, apdb: Apdb, records: list[ApdbUpdateRecord], chunk: ReplicaChunk) -> None: 

107 # Docstring inherited. 

108 assert isinstance(apdb, ApdbCassandra), "Expecting ApdbCassandra instance" 

109 apdb._storeUpdateRecords(records, chunk, store_chunk=True) 

110 

111 

112class ApdbCassandraPerMonthTestCase(ApdbCassandraTestCase): 

113 """A test case for ApdbCassandra class with per-month tables.""" 

114 

115 time_partition_tables = True 

116 time_partition_start = "2020-06-01T00:00:00" 

117 time_partition_end = "2021-06-01T00:00:00" 

118 meta_row_count = 4 

119 

120 def test_store_partition_range(self) -> None: 

121 """Test that writing to non-existing partition raises an error.""" 

122 config = self.make_instance() 

123 apdb = Apdb.from_config(config) 

124 

125 region = self.make_region() 

126 

127 # Visit time is beyond time_partition_end. 

128 visit_time = astropy.time.Time("2022-01-01", format="isot", scale="tai") 

129 catalog = makeObjectCatalog(region, 100, visit_time) 

130 with self.assertRaisesRegex(ValueError, "time partitions that do not yet exist"): 

131 apdb.store(visit_time, catalog) 

132 

133 # Writing to last partition makes a warning. 

134 visit_time = astropy.time.Time("2021-06-01", format="isot", scale="tai") 

135 catalog = makeObjectCatalog(region, 100, visit_time) 

136 with self.assertWarnsRegex(UserWarning, "Writing into the last temporal partition"): 

137 apdb.store(visit_time, catalog) 

138 

139 

140class ApdbCassandraTestCaseReplica(ApdbCassandraTestCase): 

141 """A test case with enabled replica tables.""" 

142 

143 enable_replica = True 

144 meta_row_count = 4 

145 

146 

147class ApdbCassandraTestCaseDatetimeReplica(ApdbCassandraTestCaseReplica): 

148 """A test case with datetime timestamps.""" 

149 

150 use_mjd = False 

151 

152 def setUp(self) -> None: 

153 super().setUp() 

154 # Schema for datetime case is also missing a validityTime column in 

155 # DiaObjectLast table. 

156 self.table_column_count = dict(self.table_column_count) 

157 self.table_column_count[ApdbTables.DiaObjectLast] = 4 

158 

159 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

160 if "schema_file" in kwargs: 

161 return super().make_instance(**kwargs) 

162 else: 

163 return super().make_instance(schema_file=TEST_SCHEMA_DT, **kwargs) 

164 

165 

166class ApdbSchemaUpdateCassandraTestCase(ApdbCassandraMixin, ApdbSchemaUpdateTest, unittest.TestCase): 

167 """A test case for schema updates using Cassandra backend.""" 

168 

169 time_partition_tables = False 

170 

171 def make_instance(self, **kwargs: Any) -> ApdbConfig: 

172 """Make config class instance used in all tests.""" 

173 kw = { 

174 "hosts": (self.cluster_host,), 

175 "keyspace": self.keyspace, 

176 "schema_file": TEST_SCHEMA, 

177 "ss_schema_file": TEST_SCHEMA_SSO, 

178 "time_partition_tables": self.time_partition_tables, 

179 } 

180 kw.update(kwargs) 

181 return ApdbCassandra.init_database(**kw) # type: ignore[arg-type] 

182 

183 

184class ApdbCassandraVersionCheck(cassandra_mixin.ApdbCassandraMixin, unittest.TestCase): 

185 """A test case to verify that version check happens before reading 

186 frozen configuration. 

187 """ 

188 

189 def setUp(self) -> None: 

190 super().setUp() 

191 

192 self.config = ApdbCassandra.init_database( 

193 hosts=(self.cluster_host,), 

194 keyspace=self.keyspace, 

195 schema_file=TEST_SCHEMA, 

196 ss_schema_file=TEST_SCHEMA_SSO, 

197 time_partition_tables=False, 

198 ) 

199 

200 def test_version_check(self) -> None: 

201 """Test that version check happens before reading config.""" 

202 apdb = Apdb.from_config(self.config) 

203 assert isinstance(apdb, ApdbCassandra) 

204 

205 # Store incompatible version. 

206 apdb.metadata.set(ConnectionContext.metadataSchemaVersionKey, "99.0.0", force=True) 

207 

208 # Overwrite frozen config with something that will break. 

209 apdb.metadata.set(ConnectionContext.metadataConfigKey, '{"not_a_config_key": 0}', force=True) 

210 

211 # Try again. 

212 with self.assertRaises(IncompatibleVersionError): 

213 # Need to call some actual method to initiate connection. 

214 Apdb.from_config(self.config).metadata.items() 

215 

216 

217class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

218 """Run file leak tests.""" 

219 

220 

221def setup_module(module: Any) -> None: 

222 """Configure pytest.""" 

223 lsst.utils.tests.init() 

224 

225 

226if __name__ == "__main__": 

227 lsst.utils.tests.init() 

228 unittest.main()