Coverage for tests/test_expressions.py: 22%
157 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-10-10 01:29 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <https://www.gnu.org/licenses/>.
28import datetime
29import unittest
31import astropy.time
32import sqlalchemy
33from sqlalchemy.schema import Column
35from lsst.daf.butler import (
36 ColumnTypeInfo,
37 DataCoordinate,
38 DatasetColumnTag,
39 DimensionUniverse,
40 ddl,
41 time_utils,
42)
43from lsst.daf.butler.registry.queries.expressions import make_string_expression_predicate
44from lsst.daf.butler.registry.queries.expressions.check import CheckVisitor, InspectionVisitor
45from lsst.daf.butler.registry.queries.expressions.normalForm import NormalForm, NormalFormExpression
46from lsst.daf.butler.registry.queries.expressions.parser import ParserYacc
47from lsst.daf.butler.timespan_database_representation import TimespanDatabaseRepresentation
48from lsst.daf.relation import ColumnContainer, ColumnExpression
51class FakeDatasetRecordStorageManager:
52 """Fake class for representing dataset record storage."""
54 ingestDate = Column("ingest_date")
57class ConvertExpressionToPredicateTestCase(unittest.TestCase):
58 """A test case for the make_string_expression_predicate function"""
60 ingest_date_dtype = sqlalchemy.TIMESTAMP
61 ingest_date_pytype = datetime.datetime
62 ingest_date_literal = datetime.datetime(2020, 1, 1)
64 def setUp(self):
65 self.column_types = ColumnTypeInfo(
66 timespan_cls=TimespanDatabaseRepresentation.Compound,
67 universe=DimensionUniverse(),
68 dataset_id_spec=ddl.FieldSpec("dataset_id", dtype=ddl.GUID),
69 run_key_spec=ddl.FieldSpec("run_id", dtype=sqlalchemy.BigInteger),
70 ingest_date_dtype=self.ingest_date_dtype,
71 )
73 def test_simple(self):
74 """Test with a trivial expression"""
75 self.assertEqual(
76 make_string_expression_predicate(
77 "1 > 0", self.column_types.universe.empty, column_types=self.column_types
78 )[0],
79 ColumnExpression.literal(1, dtype=int).gt(ColumnExpression.literal(0, dtype=int)),
80 )
82 def test_time(self):
83 """Test with a trivial expression including times"""
84 time_converter = time_utils.TimeConverter()
85 self.assertEqual(
86 make_string_expression_predicate(
87 "T'1970-01-01 00:00/tai' < T'2020-01-01 00:00/tai'",
88 self.column_types.universe.empty,
89 column_types=self.column_types,
90 )[0],
91 ColumnExpression.literal(time_converter.nsec_to_astropy(0), dtype=astropy.time.Time).lt(
92 ColumnExpression.literal(
93 time_converter.nsec_to_astropy(1577836800000000000), dtype=astropy.time.Time
94 )
95 ),
96 )
98 def test_ingest_date(self):
99 """Test with an expression including ingest_date which is native UTC"""
100 self.assertEqual(
101 make_string_expression_predicate(
102 "ingest_date < T'2020-01-01 00:00/utc'",
103 self.column_types.universe.empty,
104 column_types=self.column_types,
105 dataset_type_name="fake",
106 )[0],
107 ColumnExpression.reference(
108 DatasetColumnTag("fake", "ingest_date"), dtype=self.ingest_date_pytype
109 ).lt(ColumnExpression.literal(self.ingest_date_literal, dtype=self.ingest_date_pytype)),
110 )
112 def test_bind(self):
113 """Test with bind parameters"""
114 self.assertEqual(
115 make_string_expression_predicate(
116 ":a > :b OR :t in (:x, :y, :z)",
117 self.column_types.universe.empty,
118 column_types=self.column_types,
119 bind={"a": 1, "b": 2, "t": 0, "x": 10, "y": 20, "z": 30},
120 )[0],
121 ColumnExpression.literal(1, dtype=int)
122 .gt(ColumnExpression.literal(2, dtype=int))
123 .logical_or(
124 ColumnContainer.sequence(
125 [
126 ColumnExpression.literal(10, dtype=int),
127 ColumnExpression.literal(20, dtype=int),
128 ColumnExpression.literal(30, dtype=int),
129 ],
130 dtype=int,
131 ).contains(ColumnExpression.literal(0, dtype=int))
132 ),
133 )
135 def test_bind_list(self):
136 """Test with bind parameter which is list/tuple/set inside IN rhs."""
137 self.assertEqual(
138 make_string_expression_predicate(
139 ":a > :b OR :t in (:x)",
140 self.column_types.universe.empty,
141 column_types=self.column_types,
142 bind={"a": 1, "b": 2, "t": 0, "x": (10, 20, 30)},
143 )[0],
144 ColumnExpression.literal(1, dtype=int)
145 .gt(ColumnExpression.literal(2, dtype=int))
146 .logical_or(
147 ColumnContainer.sequence(
148 [
149 ColumnExpression.literal(10, dtype=int),
150 ColumnExpression.literal(20, dtype=int),
151 ColumnExpression.literal(30, dtype=int),
152 ],
153 dtype=int,
154 ).contains(
155 ColumnExpression.literal(0, dtype=int),
156 )
157 ),
158 )
159 # Couple of bound variables inside IN() with different combinations
160 # of scalars and list.
161 self.assertEqual(
162 make_string_expression_predicate(
163 ":a > :b OR :t in (:x, :y)",
164 self.column_types.universe.empty,
165 column_types=self.column_types,
166 bind={"a": 1, "b": 2, "t": 0, "x": 10, "y": 20},
167 )[0],
168 ColumnExpression.literal(1, dtype=int)
169 .gt(ColumnExpression.literal(2, dtype=int))
170 .logical_or(
171 ColumnContainer.sequence(
172 [
173 ColumnExpression.literal(10, dtype=int),
174 ColumnExpression.literal(20, dtype=int),
175 ],
176 dtype=int,
177 ).contains(
178 ColumnExpression.literal(0, dtype=int),
179 )
180 ),
181 )
182 self.assertEqual(
183 make_string_expression_predicate(
184 ":a > :b OR :t in (:x, :y)",
185 self.column_types.universe.empty,
186 column_types=self.column_types,
187 bind={"a": 1, "b": 2, "t": 0, "x": [10, 30], "y": 20},
188 )[0],
189 ColumnExpression.literal(1, dtype=int)
190 .gt(ColumnExpression.literal(2, dtype=int))
191 .logical_or(
192 ColumnContainer.sequence(
193 [
194 ColumnExpression.literal(10, dtype=int),
195 ColumnExpression.literal(30, dtype=int),
196 ColumnExpression.literal(20, dtype=int),
197 ],
198 dtype=int,
199 ).contains(
200 ColumnExpression.literal(0, dtype=int),
201 )
202 ),
203 )
204 self.assertEqual(
205 make_string_expression_predicate(
206 ":a > :b OR :t in (:x, :y)",
207 self.column_types.universe.empty,
208 column_types=self.column_types,
209 bind={"a": 1, "b": 2, "t": 0, "x": (10, 30), "y": {20}},
210 )[0],
211 ColumnExpression.literal(1, dtype=int)
212 .gt(ColumnExpression.literal(2, dtype=int))
213 .logical_or(
214 ColumnContainer.sequence(
215 [
216 ColumnExpression.literal(10, dtype=int),
217 ColumnExpression.literal(30, dtype=int),
218 ColumnExpression.literal(20, dtype=int),
219 ],
220 dtype=int,
221 ).contains(ColumnExpression.literal(0, dtype=int))
222 ),
223 )
226class ConvertExpressionToPredicateTestCaseAstropy(ConvertExpressionToPredicateTestCase):
227 """A test case for the make_string_expression_predicate function with
228 ingest_date defined as nanoseconds.
229 """
231 ingest_date_dtype = ddl.AstropyTimeNsecTai
232 ingest_date_pytype = astropy.time.Time
233 ingest_date_literal = astropy.time.Time(datetime.datetime(2020, 1, 1), scale="utc")
236class InspectionVisitorTestCase(unittest.TestCase):
237 """Tests for InspectionVisitor class."""
239 def test_simple(self):
240 """Test for simple expressions"""
241 universe = DimensionUniverse()
242 parser = ParserYacc()
244 tree = parser.parse("instrument = 'LSST'")
245 bind = {}
246 summary = tree.visit(InspectionVisitor(universe, bind))
247 self.assertEqual(summary.dimensions, {"instrument"})
248 self.assertFalse(summary.columns)
249 self.assertFalse(summary.hasIngestDate)
250 self.assertEqual(summary.dataIdKey, universe["instrument"])
251 self.assertEqual(summary.dataIdValue, "LSST")
253 tree = parser.parse("instrument != 'LSST'")
254 summary = tree.visit(InspectionVisitor(universe, bind))
255 self.assertEqual(summary.dimensions, {"instrument"})
256 self.assertFalse(summary.columns)
257 self.assertIsNone(summary.dataIdKey)
258 self.assertIsNone(summary.dataIdValue)
260 tree = parser.parse("instrument = 'LSST' AND visit = 1")
261 summary = tree.visit(InspectionVisitor(universe, bind))
262 self.assertEqual(summary.dimensions, {"instrument", "visit", "band", "physical_filter", "day_obs"})
263 self.assertFalse(summary.columns)
264 self.assertIsNone(summary.dataIdKey)
265 self.assertIsNone(summary.dataIdValue)
267 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = 'x'")
268 summary = tree.visit(InspectionVisitor(universe, bind))
269 self.assertEqual(
270 summary.dimensions, {"instrument", "visit", "band", "physical_filter", "skymap", "day_obs"}
271 )
272 self.assertFalse(summary.columns)
273 self.assertIsNone(summary.dataIdKey)
274 self.assertIsNone(summary.dataIdValue)
276 tree = parser.parse("instrument = 'LSST' AND GLOB(group, 'prefix#*')")
277 summary = tree.visit(InspectionVisitor(universe, bind))
278 self.assertEqual(summary.dimensions, {"instrument", "group"})
279 self.assertFalse(summary.columns)
280 self.assertIsNone(summary.dataIdKey)
281 self.assertIsNone(summary.dataIdValue)
283 def test_bind(self):
284 """Test for simple expressions with binds."""
285 universe = DimensionUniverse()
286 parser = ParserYacc()
288 tree = parser.parse("instrument = instr")
289 bind = {"instr": "LSST"}
290 summary = tree.visit(InspectionVisitor(universe, bind))
291 self.assertEqual(summary.dimensions, {"instrument"})
292 self.assertFalse(summary.hasIngestDate)
293 self.assertEqual(summary.dataIdKey, universe["instrument"])
294 self.assertEqual(summary.dataIdValue, "LSST")
296 tree = parser.parse("instrument != instr")
297 self.assertEqual(summary.dimensions, {"instrument"})
298 summary = tree.visit(InspectionVisitor(universe, bind))
299 self.assertIsNone(summary.dataIdKey)
300 self.assertIsNone(summary.dataIdValue)
302 tree = parser.parse("instrument = instr AND visit = visit_id")
303 bind = {"instr": "LSST", "visit_id": 1}
304 summary = tree.visit(InspectionVisitor(universe, bind))
305 self.assertEqual(summary.dimensions, {"instrument", "visit", "band", "physical_filter", "day_obs"})
306 self.assertIsNone(summary.dataIdKey)
307 self.assertIsNone(summary.dataIdValue)
309 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = skymap_name")
310 bind = {"instr": "LSST", "visit_id": 1, "skymap_name": "x"}
311 summary = tree.visit(InspectionVisitor(universe, bind))
312 self.assertEqual(
313 summary.dimensions, {"instrument", "visit", "band", "physical_filter", "skymap", "day_obs"}
314 )
315 self.assertIsNone(summary.dataIdKey)
316 self.assertIsNone(summary.dataIdValue)
318 def test_in(self):
319 """Test for IN expressions."""
320 universe = DimensionUniverse()
321 parser = ParserYacc()
323 tree = parser.parse("instrument IN ('LSST')")
324 bind = {}
325 summary = tree.visit(InspectionVisitor(universe, bind))
326 self.assertEqual(summary.dimensions, {"instrument"})
327 self.assertFalse(summary.hasIngestDate)
328 # we do not handle IN with a single item as `=`
329 self.assertIsNone(summary.dataIdKey)
330 self.assertIsNone(summary.dataIdValue)
332 tree = parser.parse("instrument IN (instr)")
333 bind = {"instr": "LSST"}
334 summary = tree.visit(InspectionVisitor(universe, bind))
335 self.assertEqual(summary.dimensions, {"instrument"})
336 self.assertIsNone(summary.dataIdKey)
337 self.assertIsNone(summary.dataIdValue)
339 tree = parser.parse("visit IN (1,2,3)")
340 bind = {}
341 summary = tree.visit(InspectionVisitor(universe, bind))
342 self.assertEqual(summary.dimensions, {"instrument", "visit", "band", "physical_filter", "day_obs"})
343 self.assertIsNone(summary.dataIdKey)
344 self.assertIsNone(summary.dataIdValue)
346 tree = parser.parse("visit IN (visit1, visit2, visit3)")
347 bind = {"visit1": 1, "visit2": 2, "visit3": 3}
348 summary = tree.visit(InspectionVisitor(universe, bind))
349 self.assertEqual(summary.dimensions, {"instrument", "visit", "band", "physical_filter", "day_obs"})
350 self.assertIsNone(summary.dataIdKey)
351 self.assertIsNone(summary.dataIdValue)
353 tree = parser.parse("visit IN (visits)")
354 bind = {"visits": (1, 2, 3)}
355 summary = tree.visit(InspectionVisitor(universe, bind))
356 self.assertEqual(summary.dimensions, {"instrument", "visit", "band", "physical_filter", "day_obs"})
357 self.assertIsNone(summary.dataIdKey)
358 self.assertIsNone(summary.dataIdValue)
361class CheckVisitorTestCase(unittest.TestCase):
362 """Tests for CheckVisitor class."""
364 def test_governor(self):
365 """Test with governor dimension in expression"""
366 parser = ParserYacc()
368 universe = DimensionUniverse()
369 dimensions = universe.conform(("instrument", "visit"))
370 dataId = DataCoordinate.make_empty(universe)
371 defaults = DataCoordinate.make_empty(universe)
373 # governor-only constraint
374 tree = parser.parse("instrument = 'LSST'")
375 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
376 binds = {}
377 visitor = CheckVisitor(dataId, dimensions, binds, defaults)
378 expr.visit(visitor)
380 tree = parser.parse("'LSST' = instrument")
381 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
382 binds = {}
383 visitor = CheckVisitor(dataId, dimensions, binds, defaults)
384 expr.visit(visitor)
386 # use bind for governor
387 tree = parser.parse("instrument = instr")
388 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE)
389 binds = {"instr": "LSST"}
390 visitor = CheckVisitor(dataId, dimensions, binds, defaults)
391 expr.visit(visitor)
394if __name__ == "__main__":
395 unittest.main()