Coverage for tests/test_expressions.py: 19%
145 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-02 14:18 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-02 14:18 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import datetime
23import unittest
25import astropy.time
26import sqlalchemy
27from lsst.daf.butler import (
28 ColumnTypeInfo,
29 DataCoordinate,
30 DatasetColumnTag,
31 DimensionUniverse,
32 TimespanDatabaseRepresentation,
33 ddl,
34 time_utils,
35)
36from lsst.daf.butler.registry.queries.expressions import make_string_expression_predicate
37from lsst.daf.butler.registry.queries.expressions.check import CheckVisitor, InspectionVisitor
38from lsst.daf.butler.registry.queries.expressions.normalForm import NormalForm, NormalFormExpression
39from lsst.daf.butler.registry.queries.expressions.parser import ParserYacc
40from lsst.daf.relation import ColumnContainer, ColumnExpression
41from sqlalchemy.schema import Column
44class FakeDatasetRecordStorageManager:
45 ingestDate = Column("ingest_date")
48class ConvertExpressionToPredicateTestCase(unittest.TestCase):
49 """A test case for the make_string_expression_predicate function"""
51 def setUp(self):
52 self.column_types = ColumnTypeInfo(
53 timespan_cls=TimespanDatabaseRepresentation.Compound,
54 universe=DimensionUniverse(),
55 dataset_id_spec=ddl.FieldSpec("dataset_id", dtype=ddl.GUID),
56 run_key_spec=ddl.FieldSpec("run_id", dtype=sqlalchemy.BigInteger),
57 )
59 def test_simple(self):
60 """Test with a trivial expression"""
61 self.assertEqual(
62 make_string_expression_predicate("1 > 0", self.column_types.universe.empty)[0],
63 ColumnExpression.literal(1, dtype=int).gt(ColumnExpression.literal(0, dtype=int)),
64 )
66 def test_time(self):
67 """Test with a trivial expression including times"""
68 time_converter = time_utils.TimeConverter()
69 self.assertEqual(
70 make_string_expression_predicate(
71 "T'1970-01-01 00:00/tai' < T'2020-01-01 00:00/tai'", self.column_types.universe.empty
72 )[0],
73 ColumnExpression.literal(time_converter.nsec_to_astropy(0), dtype=astropy.time.Time).lt(
74 ColumnExpression.literal(
75 time_converter.nsec_to_astropy(1577836800000000000), dtype=astropy.time.Time
76 )
77 ),
78 )
80 def test_ingest_date(self):
81 """Test with an expression including ingest_date which is native UTC"""
82 self.assertEqual(
83 make_string_expression_predicate(
84 "ingest_date < T'2020-01-01 00:00/utc'",
85 self.column_types.universe.empty,
86 dataset_type_name="fake",
87 )[0],
88 ColumnExpression.reference(DatasetColumnTag("fake", "ingest_date"), dtype=datetime.datetime).lt(
89 ColumnExpression.literal(datetime.datetime(2020, 1, 1), dtype=datetime.datetime)
90 ),
91 )
93 def test_bind(self):
94 """Test with bind parameters"""
96 self.assertEqual(
97 make_string_expression_predicate(
98 "a > b OR t in (x, y, z)",
99 self.column_types.universe.empty,
100 bind={"a": 1, "b": 2, "t": 0, "x": 10, "y": 20, "z": 30},
101 )[0],
102 ColumnExpression.literal(1, dtype=int)
103 .gt(ColumnExpression.literal(2, dtype=int))
104 .logical_or(
105 ColumnContainer.sequence(
106 [
107 ColumnExpression.literal(10, dtype=int),
108 ColumnExpression.literal(20, dtype=int),
109 ColumnExpression.literal(30, dtype=int),
110 ],
111 dtype=int,
112 ).contains(ColumnExpression.literal(0, dtype=int))
113 ),
114 )
116 def test_bind_list(self):
117 """Test with bind parameter which is list/tuple/set inside IN rhs."""
119 self.assertEqual(
120 make_string_expression_predicate(
121 "a > b OR t in (x)",
122 self.column_types.universe.empty,
123 bind={"a": 1, "b": 2, "t": 0, "x": (10, 20, 30)},
124 )[0],
125 ColumnExpression.literal(1, dtype=int)
126 .gt(ColumnExpression.literal(2, dtype=int))
127 .logical_or(
128 ColumnContainer.sequence(
129 [
130 ColumnExpression.literal(10, dtype=int),
131 ColumnExpression.literal(20, dtype=int),
132 ColumnExpression.literal(30, dtype=int),
133 ],
134 dtype=int,
135 ).contains(
136 ColumnExpression.literal(0, dtype=int),
137 )
138 ),
139 )
140 # Couple of bound variables inside IN() with different combinations
141 # of scalars and list.
142 self.assertEqual(
143 make_string_expression_predicate(
144 "a > b OR t in (x, y)",
145 self.column_types.universe.empty,
146 bind={"a": 1, "b": 2, "t": 0, "x": 10, "y": 20},
147 )[0],
148 ColumnExpression.literal(1, dtype=int)
149 .gt(ColumnExpression.literal(2, dtype=int))
150 .logical_or(
151 ColumnContainer.sequence(
152 [
153 ColumnExpression.literal(10, dtype=int),
154 ColumnExpression.literal(20, dtype=int),
155 ],
156 dtype=int,
157 ).contains(
158 ColumnExpression.literal(0, dtype=int),
159 )
160 ),
161 )
162 self.assertEqual(
163 make_string_expression_predicate(
164 "a > b OR t in (x, y)",
165 self.column_types.universe.empty,
166 bind={"a": 1, "b": 2, "t": 0, "x": [10, 30], "y": 20},
167 )[0],
168 ColumnExpression.literal(1, dtype=int)
169 .gt(ColumnExpression.literal(2, dtype=int))
170 .logical_or(
171 ColumnContainer.sequence(
172 [
173 ColumnExpression.literal(10, dtype=int),
174 ColumnExpression.literal(30, dtype=int),
175 ColumnExpression.literal(20, dtype=int),
176 ],
177 dtype=int,
178 ).contains(
179 ColumnExpression.literal(0, dtype=int),
180 )
181 ),
182 )
183 self.assertEqual(
184 make_string_expression_predicate(
185 "a > b OR t in (x, y)",
186 self.column_types.universe.empty,
187 bind={"a": 1, "b": 2, "t": 0, "x": (10, 30), "y": {20}},
188 )[0],
189 ColumnExpression.literal(1, dtype=int)
190 .gt(ColumnExpression.literal(2, dtype=int))
191 .logical_or(
192 ColumnContainer.sequence(
193 [
194 ColumnExpression.literal(10, dtype=int),
195 ColumnExpression.literal(30, dtype=int),
196 ColumnExpression.literal(20, dtype=int),
197 ],
198 dtype=int,
199 ).contains(ColumnExpression.literal(0, dtype=int))
200 ),
201 )
204class InspectionVisitorTestCase(unittest.TestCase):
205 """Tests for InspectionVisitor class."""
207 def test_simple(self):
208 """Test for simple expressions"""
210 universe = DimensionUniverse()
211 parser = ParserYacc()
213 tree = parser.parse("instrument = 'LSST'")
214 bind = {}
215 summary = tree.visit(InspectionVisitor(universe, bind))
216 self.assertEqual(summary.dimensions.names, {"instrument"})
217 self.assertFalse(summary.columns)
218 self.assertFalse(summary.hasIngestDate)
219 self.assertEqual(summary.dataIdKey, universe["instrument"])
220 self.assertEqual(summary.dataIdValue, "LSST")
222 tree = parser.parse("instrument != 'LSST'")
223 summary = tree.visit(InspectionVisitor(universe, bind))
224 self.assertEqual(summary.dimensions.names, {"instrument"})
225 self.assertFalse(summary.columns)
226 self.assertIsNone(summary.dataIdKey)
227 self.assertIsNone(summary.dataIdValue)
229 tree = parser.parse("instrument = 'LSST' AND visit = 1")
230 summary = tree.visit(InspectionVisitor(universe, bind))
231 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
232 self.assertFalse(summary.columns)
233 self.assertIsNone(summary.dataIdKey)
234 self.assertIsNone(summary.dataIdValue)
236 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = 'x'")
237 summary = tree.visit(InspectionVisitor(universe, bind))
238 self.assertEqual(
239 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"}
240 )
241 self.assertFalse(summary.columns)
242 self.assertIsNone(summary.dataIdKey)
243 self.assertIsNone(summary.dataIdValue)
245 def test_bind(self):
246 """Test for simple expressions with binds."""
248 universe = DimensionUniverse()
249 parser = ParserYacc()
251 tree = parser.parse("instrument = instr")
252 bind = {"instr": "LSST"}
253 summary = tree.visit(InspectionVisitor(universe, bind))
254 self.assertEqual(summary.dimensions.names, {"instrument"})
255 self.assertFalse(summary.hasIngestDate)
256 self.assertEqual(summary.dataIdKey, universe["instrument"])
257 self.assertEqual(summary.dataIdValue, "LSST")
259 tree = parser.parse("instrument != instr")
260 self.assertEqual(summary.dimensions.names, {"instrument"})
261 summary = tree.visit(InspectionVisitor(universe, bind))
262 self.assertIsNone(summary.dataIdKey)
263 self.assertIsNone(summary.dataIdValue)
265 tree = parser.parse("instrument = instr AND visit = visit_id")
266 bind = {"instr": "LSST", "visit_id": 1}
267 summary = tree.visit(InspectionVisitor(universe, bind))
268 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
269 self.assertIsNone(summary.dataIdKey)
270 self.assertIsNone(summary.dataIdValue)
272 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = skymap_name")
273 bind = {"instr": "LSST", "visit_id": 1, "skymap_name": "x"}
274 summary = tree.visit(InspectionVisitor(universe, bind))
275 self.assertEqual(
276 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"}
277 )
278 self.assertIsNone(summary.dataIdKey)
279 self.assertIsNone(summary.dataIdValue)
281 def test_in(self):
282 """Test for IN expressions."""
284 universe = DimensionUniverse()
285 parser = ParserYacc()
287 tree = parser.parse("instrument IN ('LSST')")
288 bind = {}
289 summary = tree.visit(InspectionVisitor(universe, bind))
290 self.assertEqual(summary.dimensions.names, {"instrument"})
291 self.assertFalse(summary.hasIngestDate)
292 # we do not handle IN with a single item as `=`
293 self.assertIsNone(summary.dataIdKey)
294 self.assertIsNone(summary.dataIdValue)
296 tree = parser.parse("instrument IN (instr)")
297 bind = {"instr": "LSST"}
298 summary = tree.visit(InspectionVisitor(universe, bind))
299 self.assertEqual(summary.dimensions.names, {"instrument"})
300 self.assertIsNone(summary.dataIdKey)
301 self.assertIsNone(summary.dataIdValue)
303 tree = parser.parse("visit IN (1,2,3)")
304 bind = {}
305 summary = tree.visit(InspectionVisitor(universe, bind))
306 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
307 self.assertIsNone(summary.dataIdKey)
308 self.assertIsNone(summary.dataIdValue)
310 tree = parser.parse("visit IN (visit1, visit2, visit3)")
311 bind = {"visit1": 1, "visit2": 2, "visit3": 3}
312 summary = tree.visit(InspectionVisitor(universe, bind))
313 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
314 self.assertIsNone(summary.dataIdKey)
315 self.assertIsNone(summary.dataIdValue)
317 tree = parser.parse("visit IN (visits)")
318 bind = {"visits": (1, 2, 3)}
319 summary = tree.visit(InspectionVisitor(universe, bind))
320 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"})
321 self.assertIsNone(summary.dataIdKey)
322 self.assertIsNone(summary.dataIdValue)
325class CheckVisitorTestCase(unittest.TestCase):
326 """Tests for CheckVisitor class."""
328 def test_governor(self):
329 """Test with governor dimension in expression"""
331 parser = ParserYacc()
333 universe = DimensionUniverse()
334 graph = universe.extract(("instrument", "visit"))
335 dataId = DataCoordinate.makeEmpty(universe)
336 defaults = DataCoordinate.makeEmpty(universe)
338 # governor-only constraint
339 tree = parser.parse("instrument = 'LSST'")
340 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
341 binds = {}
342 visitor = CheckVisitor(dataId, graph, binds, defaults)
343 expr.visit(visitor)
345 tree = parser.parse("'LSST' = instrument")
346 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
347 binds = {}
348 visitor = CheckVisitor(dataId, graph, binds, defaults)
349 expr.visit(visitor)
351 # use bind for governor
352 tree = parser.parse("instrument = instr")
353 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
354 binds = {"instr": "LSST"}
355 visitor = CheckVisitor(dataId, graph, binds, defaults)
356 expr.visit(visitor)
359if __name__ == "__main__": 359 ↛ 360line 359 didn't jump to line 360, because the condition on line 359 was never true
360 unittest.main()