Coverage for tests/test_expressions.py: 13%

188 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-05 10:36 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24from lsst.daf.butler import DataCoordinate, DimensionUniverse 

25from lsst.daf.butler.core import NamedKeyDict, TimespanDatabaseRepresentation 

26from lsst.daf.butler.registry.queries._structs import QueryColumns 

27from lsst.daf.butler.registry.queries.expressions import ( 

28 CheckVisitor, 

29 InspectionVisitor, 

30 NormalForm, 

31 NormalFormExpression, 

32 ParserYacc, 

33 convertExpressionToSql, 

34) 

35from sqlalchemy.dialects import postgresql, sqlite 

36from sqlalchemy.schema import Column 

37 

38 

39class FakeDatasetRecordStorageManager: 

40 ingestDate = Column("ingest_date") 

41 

42 

43class ConvertExpressionToSqlTestCase(unittest.TestCase): 

44 """A test case for convertExpressionToSql method""" 

45 

46 def setUp(self): 

47 self.universe = DimensionUniverse() 

48 

49 def test_simple(self): 

50 """Test with a trivial expression""" 

51 

52 parser = ParserYacc() 

53 tree = parser.parse("1 > 0") 

54 self.assertIsNotNone(tree) 

55 

56 columns = QueryColumns() 

57 elements = NamedKeyDict() 

58 column_element = convertExpressionToSql( 

59 tree, self.universe, columns, elements, {}, TimespanDatabaseRepresentation.Compound 

60 ) 

61 self.assertEqual(str(column_element.compile()), ":param_1 > :param_2") 

62 self.assertEqual(str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 0") 

63 

64 def test_time(self): 

65 """Test with a trivial expression including times""" 

66 

67 parser = ParserYacc() 

68 tree = parser.parse("T'1970-01-01 00:00/tai' < T'2020-01-01 00:00/tai'") 

69 self.assertIsNotNone(tree) 

70 

71 columns = QueryColumns() 

72 elements = NamedKeyDict() 

73 column_element = convertExpressionToSql( 

74 tree, self.universe, columns, elements, {}, TimespanDatabaseRepresentation.Compound 

75 ) 

76 self.assertEqual(str(column_element.compile()), ":param_1 < :param_2") 

77 self.assertEqual( 

78 str(column_element.compile(compile_kwargs={"literal_binds": True})), "0 < 1577836800000000000" 

79 ) 

80 

81 def test_ingest_date(self): 

82 """Test with an expression including ingest_date which is native UTC""" 

83 

84 parser = ParserYacc() 

85 tree = parser.parse("ingest_date < T'2020-01-01 00:00/utc'") 

86 self.assertIsNotNone(tree) 

87 

88 columns = QueryColumns() 

89 columns.datasets = FakeDatasetRecordStorageManager() 

90 elements = NamedKeyDict() 

91 column_element = convertExpressionToSql( 

92 tree, self.universe, columns, elements, {}, TimespanDatabaseRepresentation.Compound 

93 ) 

94 

95 # render it, needs specific dialect to convert column to expression 

96 dialect = postgresql.dialect() 

97 self.assertEqual(str(column_element.compile(dialect=dialect)), "ingest_date < TIMESTAMP %(param_1)s") 

98 self.assertEqual( 

99 str(column_element.compile(dialect=dialect, compile_kwargs={"literal_binds": True})), 

100 "ingest_date < TIMESTAMP '2020-01-01 00:00:00.000000'", 

101 ) 

102 

103 dialect = sqlite.dialect() 

104 self.assertEqual(str(column_element.compile(dialect=dialect)), "datetime(ingest_date) < datetime(?)") 

105 self.assertEqual( 

106 str(column_element.compile(dialect=dialect, compile_kwargs={"literal_binds": True})), 

107 "datetime(ingest_date) < datetime('2020-01-01 00:00:00.000000')", 

108 ) 

109 

110 def test_bind(self): 

111 """Test with bind parameters""" 

112 

113 parser = ParserYacc() 

114 tree = parser.parse("a > b OR t in (x, y, z)") 

115 self.assertIsNotNone(tree) 

116 

117 columns = QueryColumns() 

118 elements = NamedKeyDict() 

119 column_element = convertExpressionToSql( 

120 tree, 

121 self.universe, 

122 columns, 

123 elements, 

124 {"a": 1, "b": 2, "t": 0, "x": 10, "y": 20, "z": 30}, 

125 TimespanDatabaseRepresentation.Compound, 

126 ) 

127 self.assertEqual( 

128 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)" 

129 ) 

130 self.assertEqual( 

131 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 20, 30)" 

132 ) 

133 

134 def test_bind_list(self): 

135 """Test with bind parameter which is list/tuple/set inside IN rhs.""" 

136 

137 parser = ParserYacc() 

138 columns = QueryColumns() 

139 elements = NamedKeyDict() 

140 

141 # Single bound variable inside IN() 

142 tree = parser.parse("a > b OR t in (x)") 

143 self.assertIsNotNone(tree) 

144 column_element = convertExpressionToSql( 

145 tree, 

146 self.universe, 

147 columns, 

148 elements, 

149 {"a": 1, "b": 2, "t": 0, "x": (10, 20, 30)}, 

150 TimespanDatabaseRepresentation.Compound, 

151 ) 

152 self.assertEqual( 

153 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)" 

154 ) 

155 self.assertEqual( 

156 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 20, 30)" 

157 ) 

158 

159 # Couple of bound variables inside IN() with different combinations 

160 # of scalars and list. 

161 tree = parser.parse("a > b OR t in (x, y)") 

162 self.assertIsNotNone(tree) 

163 column_element = convertExpressionToSql( 

164 tree, 

165 self.universe, 

166 columns, 

167 elements, 

168 {"a": 1, "b": 2, "t": 0, "x": 10, "y": 20}, 

169 TimespanDatabaseRepresentation.Compound, 

170 ) 

171 self.assertEqual( 

172 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5)" 

173 ) 

174 self.assertEqual( 

175 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 20)" 

176 ) 

177 

178 column_element = convertExpressionToSql( 

179 tree, 

180 self.universe, 

181 columns, 

182 elements, 

183 {"a": 1, "b": 2, "t": 0, "x": [10, 30], "y": 20}, 

184 TimespanDatabaseRepresentation.Compound, 

185 ) 

186 self.assertEqual( 

187 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)" 

188 ) 

189 self.assertEqual( 

190 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 30, 20)" 

191 ) 

192 

193 column_element = convertExpressionToSql( 

194 tree, 

195 self.universe, 

196 columns, 

197 elements, 

198 {"a": 1, "b": 2, "t": 0, "x": (10, 30), "y": {20}}, 

199 TimespanDatabaseRepresentation.Compound, 

200 ) 

201 self.assertEqual( 

202 str(column_element.compile()), 

203 ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)", 

204 ) 

205 self.assertEqual( 

206 str(column_element.compile(compile_kwargs={"literal_binds": True})), 

207 "1 > 2 OR 0 IN (10, 30, 20)", 

208 ) 

209 

210 

211class InspectionVisitorTestCase(unittest.TestCase): 

212 """Tests for InspectionVisitor class.""" 

213 

214 def test_simple(self): 

215 """Test for simple expressions""" 

216 

217 universe = DimensionUniverse() 

218 parser = ParserYacc() 

219 

220 tree = parser.parse("instrument = 'LSST'") 

221 bind = {} 

222 summary = tree.visit(InspectionVisitor(universe, bind)) 

223 self.assertEqual(summary.dimensions.names, {"instrument"}) 

224 self.assertFalse(summary.columns) 

225 self.assertFalse(summary.hasIngestDate) 

226 self.assertEqual(summary.dataIdKey, universe["instrument"]) 

227 self.assertEqual(summary.dataIdValue, "LSST") 

228 

229 tree = parser.parse("instrument != 'LSST'") 

230 summary = tree.visit(InspectionVisitor(universe, bind)) 

231 self.assertEqual(summary.dimensions.names, {"instrument"}) 

232 self.assertFalse(summary.columns) 

233 self.assertIsNone(summary.dataIdKey) 

234 self.assertIsNone(summary.dataIdValue) 

235 

236 tree = parser.parse("instrument = 'LSST' AND visit = 1") 

237 summary = tree.visit(InspectionVisitor(universe, bind)) 

238 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

239 self.assertFalse(summary.columns) 

240 self.assertIsNone(summary.dataIdKey) 

241 self.assertIsNone(summary.dataIdValue) 

242 

243 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = 'x'") 

244 summary = tree.visit(InspectionVisitor(universe, bind)) 

245 self.assertEqual( 

246 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"} 

247 ) 

248 self.assertFalse(summary.columns) 

249 self.assertIsNone(summary.dataIdKey) 

250 self.assertIsNone(summary.dataIdValue) 

251 

252 def test_bind(self): 

253 """Test for simple expressions with binds.""" 

254 

255 universe = DimensionUniverse() 

256 parser = ParserYacc() 

257 

258 tree = parser.parse("instrument = instr") 

259 bind = {"instr": "LSST"} 

260 summary = tree.visit(InspectionVisitor(universe, bind)) 

261 self.assertEqual(summary.dimensions.names, {"instrument"}) 

262 self.assertFalse(summary.hasIngestDate) 

263 self.assertEqual(summary.dataIdKey, universe["instrument"]) 

264 self.assertEqual(summary.dataIdValue, "LSST") 

265 

266 tree = parser.parse("instrument != instr") 

267 self.assertEqual(summary.dimensions.names, {"instrument"}) 

268 summary = tree.visit(InspectionVisitor(universe, bind)) 

269 self.assertIsNone(summary.dataIdKey) 

270 self.assertIsNone(summary.dataIdValue) 

271 

272 tree = parser.parse("instrument = instr AND visit = visit_id") 

273 bind = {"instr": "LSST", "visit_id": 1} 

274 summary = tree.visit(InspectionVisitor(universe, bind)) 

275 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

276 self.assertIsNone(summary.dataIdKey) 

277 self.assertIsNone(summary.dataIdValue) 

278 

279 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = skymap_name") 

280 bind = {"instr": "LSST", "visit_id": 1, "skymap_name": "x"} 

281 summary = tree.visit(InspectionVisitor(universe, bind)) 

282 self.assertEqual( 

283 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"} 

284 ) 

285 self.assertIsNone(summary.dataIdKey) 

286 self.assertIsNone(summary.dataIdValue) 

287 

288 def test_in(self): 

289 """Test for IN expressions.""" 

290 

291 universe = DimensionUniverse() 

292 parser = ParserYacc() 

293 

294 tree = parser.parse("instrument IN ('LSST')") 

295 bind = {} 

296 summary = tree.visit(InspectionVisitor(universe, bind)) 

297 self.assertEqual(summary.dimensions.names, {"instrument"}) 

298 self.assertFalse(summary.hasIngestDate) 

299 # we do not handle IN with a single item as `=` 

300 self.assertIsNone(summary.dataIdKey) 

301 self.assertIsNone(summary.dataIdValue) 

302 

303 tree = parser.parse("instrument IN (instr)") 

304 bind = {"instr": "LSST"} 

305 summary = tree.visit(InspectionVisitor(universe, bind)) 

306 self.assertEqual(summary.dimensions.names, {"instrument"}) 

307 self.assertIsNone(summary.dataIdKey) 

308 self.assertIsNone(summary.dataIdValue) 

309 

310 tree = parser.parse("visit IN (1,2,3)") 

311 bind = {} 

312 summary = tree.visit(InspectionVisitor(universe, bind)) 

313 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

314 self.assertIsNone(summary.dataIdKey) 

315 self.assertIsNone(summary.dataIdValue) 

316 

317 tree = parser.parse("visit IN (visit1, visit2, visit3)") 

318 bind = {"visit1": 1, "visit2": 2, "visit3": 3} 

319 summary = tree.visit(InspectionVisitor(universe, bind)) 

320 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

321 self.assertIsNone(summary.dataIdKey) 

322 self.assertIsNone(summary.dataIdValue) 

323 

324 tree = parser.parse("visit IN (visits)") 

325 bind = {"visits": (1, 2, 3)} 

326 summary = tree.visit(InspectionVisitor(universe, bind)) 

327 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

328 self.assertIsNone(summary.dataIdKey) 

329 self.assertIsNone(summary.dataIdValue) 

330 

331 

332class CheckVisitorTestCase(unittest.TestCase): 

333 """Tests for CheckVisitor class.""" 

334 

335 def test_governor(self): 

336 """Test with governor dimension in expression""" 

337 

338 parser = ParserYacc() 

339 

340 universe = DimensionUniverse() 

341 graph = universe.extract(("instrument", "visit")) 

342 dataId = DataCoordinate.makeEmpty(universe) 

343 defaults = DataCoordinate.makeEmpty(universe) 

344 

345 # governor-only constraint 

346 tree = parser.parse("instrument = 'LSST'") 

347 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE) 

348 binds = {} 

349 visitor = CheckVisitor(dataId, graph, binds, defaults) 

350 expr.visit(visitor) 

351 

352 tree = parser.parse("'LSST' = instrument") 

353 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE) 

354 binds = {} 

355 visitor = CheckVisitor(dataId, graph, binds, defaults) 

356 expr.visit(visitor) 

357 

358 # use bind for governor 

359 tree = parser.parse("instrument = instr") 

360 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE) 

361 binds = {"instr": "LSST"} 

362 visitor = CheckVisitor(dataId, graph, binds, defaults) 

363 expr.visit(visitor) 

364 

365 

366if __name__ == "__main__": 366 ↛ 367line 366 didn't jump to line 367, because the condition on line 366 was never true

367 unittest.main()