Coverage for tests/test_expressions.py: 13%
188 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-09 02:51 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-09 02:51 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24from lsst.daf.butler import DataCoordinate, DimensionUniverse
25from lsst.daf.butler.core import NamedKeyDict, TimespanDatabaseRepresentation
26from lsst.daf.butler.registry.queries._structs import QueryColumns
27from lsst.daf.butler.registry.queries.expressions import (
28 CheckVisitor,
29 InspectionVisitor,
30 NormalForm,
31 NormalFormExpression,
32 ParserYacc,
33 convertExpressionToSql,
34)
35from sqlalchemy.dialects import postgresql, sqlite
36from sqlalchemy.schema import Column
39class FakeDatasetRecordStorageManager:
40 ingestDate = Column("ingest_date")
43class ConvertExpressionToSqlTestCase(unittest.TestCase):
44 """A test case for convertExpressionToSql method"""
46 def setUp(self):
47 self.universe = DimensionUniverse()
49 def test_simple(self):
50 """Test with a trivial expression"""
52 parser = ParserYacc()
53 tree = parser.parse("1 > 0")
54 self.assertIsNotNone(tree)
56 columns = QueryColumns()
57 elements = NamedKeyDict()
58 column_element = convertExpressionToSql(
59 tree, self.universe, columns, elements, {}, TimespanDatabaseRepresentation.Compound
60 )
61 self.assertEqual(str(column_element.compile()), ":param_1 > :param_2")
62 self.assertEqual(str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 0")
64 def test_time(self):
65 """Test with a trivial expression including times"""
67 parser = ParserYacc()
68 tree = parser.parse("T'1970-01-01 00:00/tai' < T'2020-01-01 00:00/tai'")
69 self.assertIsNotNone(tree)
71 columns = QueryColumns()
72 elements = NamedKeyDict()
73 column_element = convertExpressionToSql(
74 tree, self.universe, columns, elements, {}, TimespanDatabaseRepresentation.Compound
75 )
76 self.assertEqual(str(column_element.compile()), ":param_1 < :param_2")
77 self.assertEqual(
78 str(column_element.compile(compile_kwargs={"literal_binds": True})), "0 < 1577836800000000000"
79 )
81 def test_ingest_date(self):
82 """Test with an expression including ingest_date which is native UTC"""
84 parser = ParserYacc()
85 tree = parser.parse("ingest_date < T'2020-01-01 00:00/utc'")
86 self.assertIsNotNone(tree)
88 columns = QueryColumns()
89 columns.datasets = FakeDatasetRecordStorageManager()
90 elements = NamedKeyDict()
91 column_element = convertExpressionToSql(
92 tree, self.universe, columns, elements, {}, TimespanDatabaseRepresentation.Compound
93 )
95 # render it, needs specific dialect to convert column to expression
96 dialect = postgresql.dialect()
97 self.assertEqual(str(column_element.compile(dialect=dialect)), "ingest_date < TIMESTAMP %(param_1)s")
98 self.assertEqual(
99 str(column_element.compile(dialect=dialect, compile_kwargs={"literal_binds": True})),
100 "ingest_date < TIMESTAMP '2020-01-01 00:00:00.000000'",
101 )
103 dialect = sqlite.dialect()
104 self.assertEqual(str(column_element.compile(dialect=dialect)), "datetime(ingest_date) < datetime(?)")
105 self.assertEqual(
106 str(column_element.compile(dialect=dialect, compile_kwargs={"literal_binds": True})),
107 "datetime(ingest_date) < datetime('2020-01-01 00:00:00.000000')",
108 )
110 def test_bind(self):
111 """Test with bind parameters"""
113 parser = ParserYacc()
114 tree = parser.parse("a > b OR t in (x, y, z)")
115 self.assertIsNotNone(tree)
117 columns = QueryColumns()
118 elements = NamedKeyDict()
119 column_element = convertExpressionToSql(
120 tree,
121 self.universe,
122 columns,
123 elements,
124 {"a": 1, "b": 2, "t": 0, "x": 10, "y": 20, "z": 30},
125 TimespanDatabaseRepresentation.Compound,
126 )
127 self.assertEqual(
128 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)"
129 )
130 self.assertEqual(
131 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 20, 30)"
132 )
134 def test_bind_list(self):
135 """Test with bind parameter which is list/tuple/set inside IN rhs."""
137 parser = ParserYacc()
138 columns = QueryColumns()
139 elements = NamedKeyDict()
141 # Single bound variable inside IN()
142 tree = parser.parse("a > b OR t in (x)")
143 self.assertIsNotNone(tree)
144 column_element = convertExpressionToSql(
145 tree,
146 self.universe,
147 columns,
148 elements,
149 {"a": 1, "b": 2, "t": 0, "x": (10, 20, 30)},
150 TimespanDatabaseRepresentation.Compound,
151 )
152 self.assertEqual(
153 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)"
154 )
155 self.assertEqual(
156 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 20, 30)"
157 )
159 # Couple of bound variables inside IN() with different combinations
160 # of scalars and list.
161 tree = parser.parse("a > b OR t in (x, y)")
162 self.assertIsNotNone(tree)
163 column_element = convertExpressionToSql(
164 tree,
165 self.universe,
166 columns,
167 elements,
168 {"a": 1, "b": 2, "t": 0, "x": 10, "y": 20},
169 TimespanDatabaseRepresentation.Compound,
170 )
171 self.assertEqual(
172 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5)"
173 )
174 self.assertEqual(
175 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 20)"
176 )
178 column_element = convertExpressionToSql(
179 tree,
180 self.universe,
181 columns,
182 elements,
183 {"a": 1, "b": 2, "t": 0, "x": [10, 30], "y": 20},
184 TimespanDatabaseRepresentation.Compound,
185 )
186 self.assertEqual(
187 str(column_element.compile()), ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)"
188 )
189 self.assertEqual(
190 str(column_element.compile(compile_kwargs={"literal_binds": True})), "1 > 2 OR 0 IN (10, 30, 20)"
191 )
193 column_element = convertExpressionToSql(
194 tree,
195 self.universe,
196 columns,
197 elements,
198 {"a": 1, "b": 2, "t": 0, "x": (10, 30), "y": {20}},
199 TimespanDatabaseRepresentation.Compound,
200 )
201 self.assertEqual(
202 str(column_element.compile()),
203 ":param_1 > :param_2 OR :param_3 IN (:param_4, :param_5, :param_6)",
204 )
205 self.assertEqual(
206 str(column_element.compile(compile_kwargs={"literal_binds": True})),
207 "1 > 2 OR 0 IN (10, 30, 20)",
208 )
211class InspectionVisitorTestCase(unittest.TestCase):
212 """Tests for InspectionVisitor class."""
214 def test_simple(self):
215 """Test for simple expressions"""
217 universe = DimensionUniverse()
218 parser = ParserYacc()
220 tree = parser.parse("instrument = 'LSST'")
221 bind = {}
222 summary = tree.visit(InspectionVisitor(universe, bind))
223 self.assertEqual(summary.dimensions.names, {"instrument"})
224 self.assertFalse(summary.columns)
225 self.assertFalse(summary.hasIngestDate)
226 self.assertEqual(summary.dataIdKey, universe["instrument"])
227 self.assertEqual(summary.dataIdValue, "LSST")
229 tree = parser.parse("instrument != 'LSST'")
230 summary = tree.visit(InspectionVisitor(universe, bind))
231 self.assertEqual(summary.dimensions.names, {"instrument"})
232 self.assertFalse(summary.columns)
233 self.assertIsNone(summary.dataIdKey)
234 self.assertIsNone(summary.dataIdValue)
236 tree = parser.parse("instrument = 'LSST' AND visit = 1")
237 summary = tree.visit(InspectionVisitor(universe, bind))
238 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
239 self.assertFalse(summary.columns)
240 self.assertIsNone(summary.dataIdKey)
241 self.assertIsNone(summary.dataIdValue)
243 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = 'x'")
244 summary = tree.visit(InspectionVisitor(universe, bind))
245 self.assertEqual(
246 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"}
247 )
248 self.assertFalse(summary.columns)
249 self.assertIsNone(summary.dataIdKey)
250 self.assertIsNone(summary.dataIdValue)
252 def test_bind(self):
253 """Test for simple expressions with binds."""
255 universe = DimensionUniverse()
256 parser = ParserYacc()
258 tree = parser.parse("instrument = instr")
259 bind = {"instr": "LSST"}
260 summary = tree.visit(InspectionVisitor(universe, bind))
261 self.assertEqual(summary.dimensions.names, {"instrument"})
262 self.assertFalse(summary.hasIngestDate)
263 self.assertEqual(summary.dataIdKey, universe["instrument"])
264 self.assertEqual(summary.dataIdValue, "LSST")
266 tree = parser.parse("instrument != instr")
267 self.assertEqual(summary.dimensions.names, {"instrument"})
268 summary = tree.visit(InspectionVisitor(universe, bind))
269 self.assertIsNone(summary.dataIdKey)
270 self.assertIsNone(summary.dataIdValue)
272 tree = parser.parse("instrument = instr AND visit = visit_id")
273 bind = {"instr": "LSST", "visit_id": 1}
274 summary = tree.visit(InspectionVisitor(universe, bind))
275 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
276 self.assertIsNone(summary.dataIdKey)
277 self.assertIsNone(summary.dataIdValue)
279 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = skymap_name")
280 bind = {"instr": "LSST", "visit_id": 1, "skymap_name": "x"}
281 summary = tree.visit(InspectionVisitor(universe, bind))
282 self.assertEqual(
283 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"}
284 )
285 self.assertIsNone(summary.dataIdKey)
286 self.assertIsNone(summary.dataIdValue)
288 def test_in(self):
289 """Test for IN expressions."""
291 universe = DimensionUniverse()
292 parser = ParserYacc()
294 tree = parser.parse("instrument IN ('LSST')")
295 bind = {}
296 summary = tree.visit(InspectionVisitor(universe, bind))
297 self.assertEqual(summary.dimensions.names, {"instrument"})
298 self.assertFalse(summary.hasIngestDate)
299 # we do not handle IN with a single item as `=`
300 self.assertIsNone(summary.dataIdKey)
301 self.assertIsNone(summary.dataIdValue)
303 tree = parser.parse("instrument IN (instr)")
304 bind = {"instr": "LSST"}
305 summary = tree.visit(InspectionVisitor(universe, bind))
306 self.assertEqual(summary.dimensions.names, {"instrument"})
307 self.assertIsNone(summary.dataIdKey)
308 self.assertIsNone(summary.dataIdValue)
310 tree = parser.parse("visit IN (1,2,3)")
311 bind = {}
312 summary = tree.visit(InspectionVisitor(universe, bind))
313 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
314 self.assertIsNone(summary.dataIdKey)
315 self.assertIsNone(summary.dataIdValue)
317 tree = parser.parse("visit IN (visit1, visit2, visit3)")
318 bind = {"visit1": 1, "visit2": 2, "visit3": 3}
319 summary = tree.visit(InspectionVisitor(universe, bind))
320 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
321 self.assertIsNone(summary.dataIdKey)
322 self.assertIsNone(summary.dataIdValue)
324 tree = parser.parse("visit IN (visits)")
325 bind = {"visits": (1, 2, 3)}
326 summary = tree.visit(InspectionVisitor(universe, bind))
327 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
328 self.assertIsNone(summary.dataIdKey)
329 self.assertIsNone(summary.dataIdValue)
332class CheckVisitorTestCase(unittest.TestCase):
333 """Tests for CheckVisitor class."""
335 def test_governor(self):
336 """Test with governor dimension in expression"""
338 parser = ParserYacc()
340 universe = DimensionUniverse()
341 graph = universe.extract(("instrument", "visit"))
342 dataId = DataCoordinate.makeEmpty(universe)
343 defaults = DataCoordinate.makeEmpty(universe)
345 # governor-only constraint
346 tree = parser.parse("instrument = 'LSST'")
347 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
348 binds = {}
349 visitor = CheckVisitor(dataId, graph, binds, defaults)
350 expr.visit(visitor)
352 tree = parser.parse("'LSST' = instrument")
353 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
354 binds = {}
355 visitor = CheckVisitor(dataId, graph, binds, defaults)
356 expr.visit(visitor)
358 # use bind for governor
359 tree = parser.parse("instrument = instr")
360 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
361 binds = {"instr": "LSST"}
362 visitor = CheckVisitor(dataId, graph, binds, defaults)
363 expr.visit(visitor)
366if __name__ == "__main__": 366 ↛ 367line 366 didn't jump to line 367, because the condition on line 366 was never true
367 unittest.main()