1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for ApdbSchema class. 

23""" 

24 

25import os 

26import unittest 

27 

28import lsst.afw.table as afwTable 

29from lsst.dax.apdb import ApdbSchema, make_minimal_dia_object_schema, make_minimal_dia_source_schema 

30from lsst.utils import getPackageDir 

31import lsst.utils.tests 

32from sqlalchemy import create_engine 

33 

34 

35def _make_case_conficting_dia_object_schema(): 

36 """Make schema which has column name with case mismatch. 

37 

38 Copy of make_minimal_dia_object_schema with additional column. 

39 """ 

40 schema = afwTable.SourceTable.makeMinimalSchema() 

41 schema.addField("pixelId", type='L', 

42 doc='Unique spherical pixelization identifier.') 

43 schema.addField("nDiaSources", type='L') 

44 # baseline schema has column `radecTai` 

45 schema.addField("RaDecTai", type='D') 

46 return schema 

47 

48 

49def _data_file_name(basename): 

50 """Return path name of a data file. 

51 """ 

52 return os.path.join(getPackageDir("dax_apdb"), "data", basename) 

53 

54 

55class ApdbSchemaTestCase(unittest.TestCase): 

56 """A test case for ApdbSchema class 

57 """ 

58 

59 @classmethod 

60 def setUpClass(cls): 

61 pass 

62 

63 def _assertTable(self, table, name, ncol): 

64 """validation for tables schema. 

65 

66 Parameters 

67 ---------- 

68 table : `sqlalchemy.Table` 

69 name : `str` 

70 Expected table name 

71 ncol : `int` 

72 Expected number of columns 

73 """ 

74 self.assertIsNotNone(table) 

75 self.assertEqual(table.name, name) 

76 self.assertEqual(len(table.columns), ncol) 

77 

78 def test_makeSchema(self): 

79 """Test for creating schemas. 

80 

81 Schema is defined in YAML files, some checks here depend on that 

82 configuration and will need to be updated when configuration changes. 

83 """ 

84 engine = create_engine('sqlite://') 

85 

86 # create standard (baseline) schema 

87 schema = ApdbSchema(engine=engine, 

88 dia_object_index="baseline", 

89 dia_object_nightly=False, 

90 schema_file=_data_file_name("apdb-schema.yaml")) 

91 schema.makeSchema() 

92 self._assertTable(schema.objects, "DiaObject", 92) 

93 self.assertEqual(len(schema.objects.primary_key), 2) 

94 self.assertIsNone(schema.objects_nightly) 

95 self.assertIsNone(schema.objects_last) 

96 self._assertTable(schema.sources, "DiaSource", 108) 

97 self._assertTable(schema.forcedSources, "DiaForcedSource", 7) 

98 

99 # create schema using prefix 

100 schema = ApdbSchema(engine=engine, 

101 dia_object_index="baseline", 

102 dia_object_nightly=False, 

103 schema_file=_data_file_name("apdb-schema.yaml"), 

104 prefix="Pfx") 

105 # Drop existing tables (but we don't check it here) 

106 schema.makeSchema(drop=True) 

107 self._assertTable(schema.objects, "PfxDiaObject", 92) 

108 self.assertIsNone(schema.objects_nightly) 

109 self.assertIsNone(schema.objects_last) 

110 self._assertTable(schema.sources, "PfxDiaSource", 108) 

111 self._assertTable(schema.forcedSources, "PfxDiaForcedSource", 7) 

112 

113 # use different indexing for DiaObject, need extra schema for that 

114 schema = ApdbSchema(engine=engine, 

115 dia_object_index="pix_id_iov", 

116 dia_object_nightly=False, 

117 schema_file=_data_file_name("apdb-schema.yaml"), 

118 extra_schema_file=_data_file_name("apdb-schema-extra.yaml")) 

119 schema.makeSchema(drop=True) 

120 self._assertTable(schema.objects, "DiaObject", 94) 

121 self.assertEqual(len(schema.objects.primary_key), 3) 

122 self.assertIsNone(schema.objects_nightly) 

123 self.assertIsNone(schema.objects_last) 

124 self._assertTable(schema.sources, "DiaSource", 108) 

125 self._assertTable(schema.forcedSources, "DiaForcedSource", 7) 

126 

127 # use DiaObjectLast table for DiaObject, need extra schema for that 

128 schema = ApdbSchema(engine=engine, 

129 dia_object_index="last_object_table", 

130 dia_object_nightly=False, 

131 schema_file=_data_file_name("apdb-schema.yaml"), 

132 extra_schema_file=_data_file_name("apdb-schema-extra.yaml")) 

133 schema.makeSchema(drop=True) 

134 self._assertTable(schema.objects, "DiaObject", 94) 

135 self.assertEqual(len(schema.objects.primary_key), 2) 

136 self.assertIsNone(schema.objects_nightly) 

137 self._assertTable(schema.objects_last, "DiaObjectLast", 18) 

138 self.assertEqual(len(schema.objects_last.primary_key), 2) 

139 self._assertTable(schema.sources, "DiaSource", 108) 

140 self._assertTable(schema.forcedSources, "DiaForcedSource", 7) 

141 

142 # baseline schema with nightly DiaObject 

143 schema = ApdbSchema(engine=engine, 

144 dia_object_index="baseline", 

145 dia_object_nightly=True, 

146 schema_file=_data_file_name("apdb-schema.yaml")) 

147 schema.makeSchema(drop=True) 

148 self._assertTable(schema.objects, "DiaObject", 92) 

149 self._assertTable(schema.objects_nightly, "DiaObjectNightly", 92) 

150 self.assertIsNone(schema.objects_last) 

151 self._assertTable(schema.sources, "DiaSource", 108) 

152 self._assertTable(schema.forcedSources, "DiaForcedSource", 7) 

153 

154 def test_afwSchemaCaseSensitivity(self): 

155 """Test for column case mismatch errors. 

156 

157 This is a specific test for when afw schema column names differ from 

158 APDB schem in case only which should generate exception. 

159 

160 Like all other tests this depends on the column naming in 

161 apdb-schema.yaml. 

162 """ 

163 engine = create_engine('sqlite://') 

164 

165 afw_schemas = dict(DiaObject=_make_case_conficting_dia_object_schema(), 

166 DiaSource=make_minimal_dia_source_schema()) 

167 # column case mismatch should cause exception in constructor 

168 with self.assertRaises(ValueError): 

169 ApdbSchema(engine=engine, 

170 dia_object_index="baseline", 

171 dia_object_nightly=False, 

172 schema_file=_data_file_name("apdb-schema.yaml"), 

173 column_map=_data_file_name("apdb-afw-map.yaml"), 

174 afw_schemas=afw_schemas) 

175 

176 def test_getAfwSchema(self): 

177 """Test for getAfwSchema method. 

178 

179 Schema is defined in YAML files, some checks here depend on that 

180 configuration and will need to be updated when configuration changes. 

181 """ 

182 engine = create_engine('sqlite://') 

183 

184 # create standard (baseline) schema, but use afw column map 

185 schema = ApdbSchema(engine=engine, 

186 dia_object_index="baseline", 

187 dia_object_nightly=False, 

188 schema_file=_data_file_name("apdb-schema.yaml"), 

189 column_map=_data_file_name("apdb-afw-map.yaml")) 

190 schema.makeSchema() 

191 

192 afw_schema, col_map = schema.getAfwSchema("DiaObject") 

193 self.assertEqual(len(col_map), 92) 

194 self.assertIsInstance(afw_schema, afwTable.Schema) 

195 # no BLOBs in afwTable, so count is lower 

196 self.assertEqual(afw_schema.getFieldCount(), 81) 

197 

198 afw_schema, col_map = schema.getAfwSchema("DiaSource") 

199 self.assertEqual(len(col_map), 108) 

200 self.assertIsInstance(afw_schema, afwTable.Schema) 

201 self.assertEqual(afw_schema.getFieldCount(), 108) 

202 

203 afw_schema, col_map = schema.getAfwSchema("DiaForcedSource") 

204 self.assertEqual(len(col_map), 7) 

205 self.assertIsInstance(afw_schema, afwTable.Schema) 

206 # afw table adds 4 columns compared to out standard schema 

207 self.assertEqual(afw_schema.getFieldCount(), 7+4) 

208 

209 # subset of columns 

210 afw_schema, col_map = schema.getAfwSchema("DiaObject", 

211 ["diaObjectId", "ra", "decl", "ra_decl_Cov"]) 

212 self.assertEqual(len(col_map), 4) 

213 self.assertIsInstance(afw_schema, afwTable.Schema) 

214 # one extra column exists for some reason for DiaObect in afw schema 

215 self.assertEqual(afw_schema.getFieldCount(), 5) 

216 

217 def test_getAfwSchemaWithExtras(self): 

218 """Test for getAfwSchema method using extra afw schemas. 

219 

220 Same as above but use non-default afw schemas, this adds few extra 

221 columns to the table schema 

222 """ 

223 engine = create_engine('sqlite://') 

224 

225 # create standard (baseline) schema, but use afw column map 

226 afw_schemas = dict(DiaObject=make_minimal_dia_object_schema(), 

227 DiaSource=make_minimal_dia_source_schema()) 

228 schema = ApdbSchema(engine=engine, 

229 dia_object_index="baseline", 

230 dia_object_nightly=False, 

231 schema_file=_data_file_name("apdb-schema.yaml"), 

232 column_map=_data_file_name("apdb-afw-map.yaml"), 

233 afw_schemas=afw_schemas) 

234 schema.makeSchema() 

235 

236 afw_schema, col_map = schema.getAfwSchema("DiaObject") 

237 self.assertEqual(len(col_map), 94) 

238 self.assertIsInstance(afw_schema, afwTable.Schema) 

239 # no BLOBs in afwTable, so count is lower 

240 self.assertEqual(afw_schema.getFieldCount(), 82) 

241 

242 afw_schema, col_map = schema.getAfwSchema("DiaSource") 

243 self.assertEqual(len(col_map), 108) 

244 self.assertIsInstance(afw_schema, afwTable.Schema) 

245 self.assertEqual(afw_schema.getFieldCount(), 108) 

246 

247 afw_schema, col_map = schema.getAfwSchema("DiaForcedSource") 

248 self.assertEqual(len(col_map), 7) 

249 self.assertIsInstance(afw_schema, afwTable.Schema) 

250 # afw table adds 4 columns compared to out standard schema 

251 self.assertEqual(afw_schema.getFieldCount(), 7+4) 

252 

253 # subset of columns 

254 afw_schema, col_map = schema.getAfwSchema("DiaObject", 

255 ["diaObjectId", "ra", "decl", "ra_decl_Cov"]) 

256 self.assertEqual(len(col_map), 4) 

257 self.assertIsInstance(afw_schema, afwTable.Schema) 

258 # one extra column exists for some reason for DiaObect in afw schema 

259 self.assertEqual(afw_schema.getFieldCount(), 5) 

260 

261 def test_getAfwColumns(self): 

262 """Test for getAfwColumns method. 

263 

264 Schema is defined in YAML files, some checks here depend on that 

265 configuration and will need to be updated when configuration changes. 

266 """ 

267 engine = create_engine('sqlite://') 

268 

269 # create standard (baseline) schema, but use afw column map 

270 schema = ApdbSchema(engine=engine, 

271 dia_object_index="baseline", 

272 dia_object_nightly=False, 

273 schema_file=_data_file_name("apdb-schema.yaml"), 

274 column_map=_data_file_name("apdb-afw-map.yaml")) 

275 schema.makeSchema() 

276 

277 col_map = schema.getAfwColumns("DiaObject") 

278 self.assertEqual(len(col_map), 92) 

279 # check few afw-specific names 

280 self.assertIn("id", col_map) 

281 self.assertIn("coord_ra", col_map) 

282 self.assertIn("coord_dec", col_map) 

283 

284 col_map = schema.getAfwColumns("DiaSource") 

285 self.assertEqual(len(col_map), 108) 

286 # check few afw-specific names 

287 self.assertIn("id", col_map) 

288 self.assertIn("coord_ra", col_map) 

289 self.assertIn("coord_dec", col_map) 

290 

291 

292class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

293 pass 

294 

295 

296def setup_module(module): 

297 lsst.utils.tests.init() 

298 

299 

300if __name__ == "__main__": 300 ↛ 301line 300 didn't jump to line 301, because the condition on line 300 was never true

301 lsst.utils.tests.init() 

302 unittest.main()