Coverage for tests / test_exprParserLex.py: 9%
196 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-14 23:37 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <https://www.gnu.org/licenses/>.
28"""Simple unit test for expr_parser/parserLex module."""
30import re
31import unittest
33from lsst.daf.butler.queries.expressions.parser import ParserLex, ParserLexError
36class ParserLexTestCase(unittest.TestCase):
37 """A test case for ParserLex"""
39 def _assertToken(self, token, type, value, lineno=None, lexpos=None):
40 self.assertIsNotNone(token)
41 self.assertEqual(token.type, type)
42 self.assertEqual(token.value, value)
43 if lineno is not None:
44 self.assertEqual(token.lineno, lineno)
45 if lexpos is not None:
46 self.assertEqual(token.lexpos, lexpos)
48 def setUp(self):
49 pass
51 def tearDown(self):
52 pass
54 def testInstantiate(self):
55 """Tests for making ParserLex instances"""
56 default_reflags = re.IGNORECASE | re.VERBOSE
57 lexer = ParserLex.make_lexer()
58 self.assertEqual(lexer.lexreflags, default_reflags)
60 lexer = ParserLex.make_lexer(reflags=re.DOTALL)
61 self.assertEqual(lexer.lexreflags, re.DOTALL | default_reflags)
63 def testSimpleTokens(self):
64 """Test for simple tokens"""
65 lexer = ParserLex.make_lexer()
67 lexer.input("=!= <<= >>= +-*/()")
68 self._assertToken(lexer.token(), "EQ", "=")
69 self._assertToken(lexer.token(), "NE", "!=")
70 self._assertToken(lexer.token(), "LT", "<")
71 self._assertToken(lexer.token(), "LE", "<=")
72 self._assertToken(lexer.token(), "GT", ">")
73 self._assertToken(lexer.token(), "GE", ">=")
74 self._assertToken(lexer.token(), "ADD", "+")
75 self._assertToken(lexer.token(), "SUB", "-")
76 self._assertToken(lexer.token(), "MUL", "*")
77 self._assertToken(lexer.token(), "DIV", "/")
78 self._assertToken(lexer.token(), "LPAREN", "(")
79 self._assertToken(lexer.token(), "RPAREN", ")")
80 self.assertIsNone(lexer.token())
82 def testReservedTokens(self):
83 """Test for reserved words"""
84 lexer = ParserLex.make_lexer()
86 tokens = "NOT IN OR AND OVERLAPS"
87 lexer.input(tokens)
88 for token in tokens.split():
89 self._assertToken(lexer.token(), token, token)
90 self.assertIsNone(lexer.token())
92 tokens = "not in or and overlaps"
93 lexer.input(tokens)
94 for token in tokens.split():
95 self._assertToken(lexer.token(), token.upper(), token.upper())
96 self.assertIsNone(lexer.token())
98 # not reserved
99 token = "NOTIN"
100 lexer.input(token)
101 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", token)
102 self.assertIsNone(lexer.token())
104 def testStringLiteral(self):
105 """Test for string literals"""
106 lexer = ParserLex.make_lexer()
108 lexer.input("''")
109 self._assertToken(lexer.token(), "STRING_LITERAL", "")
110 self.assertIsNone(lexer.token())
112 lexer.input("'string'")
113 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
114 self.assertIsNone(lexer.token())
116 lexer.input("'string' 'string'\n'string'")
117 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
118 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
119 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
120 self.assertIsNone(lexer.token())
122 # odd newline inside string
123 lexer.input("'string\nstring'")
124 with self.assertRaises(ParserLexError):
125 lexer.token()
127 lexer.input("'string")
128 with self.assertRaises(ParserLexError):
129 lexer.token()
131 def testNumericLiteral(self):
132 """Test for numeric literals"""
133 lexer = ParserLex.make_lexer()
135 lexer.input("0 100 999. 100.1 1e10 1e-10 1.e+20 .2E5")
136 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "0")
137 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100")
138 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "999.")
139 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100.1")
140 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e10")
141 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e-10")
142 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1.e+20")
143 self._assertToken(lexer.token(), "NUMERIC_LITERAL", ".2E5")
144 self.assertIsNone(lexer.token())
146 def testRangeLiteral(self):
147 """Test for range literals"""
148 lexer = ParserLex.make_lexer()
150 lexer.input("0..10 -10..-1 -10..10:2 0 .. 10 0 .. 10 : 2 ")
151 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None))
152 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, -1, None))
153 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, 10, 2))
154 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None))
155 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, 2))
156 self.assertIsNone(lexer.token())
158 def testTimeLiteral(self):
159 """Test for time literals"""
160 lexer = ParserLex.make_lexer()
162 # string can contain anything, lexer does not check it
163 lexer.input("T'2020-03-30' T'2020-03-30 00:00:00' T'2020-03-30T00:00:00' T'123.456' T'time'")
164 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30")
165 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30 00:00:00")
166 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30T00:00:00")
167 self._assertToken(lexer.token(), "TIME_LITERAL", "123.456")
168 self._assertToken(lexer.token(), "TIME_LITERAL", "time")
169 self.assertIsNone(lexer.token())
171 def testIdentifier(self):
172 """Test for numeric literals"""
173 lexer = ParserLex.make_lexer()
175 lexer.input("ID id _012 a_b_C")
176 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "ID")
177 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id")
178 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "_012")
179 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a_b_C")
180 self.assertIsNone(lexer.token())
182 lexer.input("a.b a.b.c _._ _._._")
183 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b")
184 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c")
185 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._")
186 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._._")
187 self.assertIsNone(lexer.token())
189 lexer.input(".id")
190 with self.assertRaises(ParserLexError):
191 lexer.token()
193 lexer.input("id.")
194 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id")
195 with self.assertRaises(ParserLexError):
196 lexer.token()
198 lexer.input("id.id.id.id")
199 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "id.id.id")
200 with self.assertRaises(ParserLexError):
201 lexer.token()
203 def testNinds(self):
204 """Test for bind names"""
205 lexer = ParserLex.make_lexer()
207 lexer.input(":name :visit :in :___")
208 self._assertToken(lexer.token(), "BIND_NAME", "name")
209 self._assertToken(lexer.token(), "BIND_NAME", "visit")
210 self._assertToken(lexer.token(), "BIND_NAME", "in")
211 self._assertToken(lexer.token(), "BIND_NAME", "___")
213 lexer.input(":1")
214 with self.assertRaises(ParserLexError):
215 lexer.token()
217 lexer.input(":.id")
218 with self.assertRaises(ParserLexError):
219 lexer.token()
221 lexer.input(":id.")
222 self._assertToken(lexer.token(), "BIND_NAME", "id")
223 with self.assertRaises(ParserLexError):
224 lexer.token()
226 def testExpression(self):
227 """Test for more or less complete expression"""
228 lexer = ParserLex.make_lexer()
230 expr = (
231 "((instrument='HSC' AND detector != 9) OR instrument='CFHT') "
232 "AND tract=8766 AND patch.cell_x > 5 AND "
233 "patch.cell_y < 4 AND band='i' "
234 "or visit IN (1..50:2)"
235 )
236 tokens = (
237 ("LPAREN", "("),
238 ("LPAREN", "("),
239 ("SIMPLE_IDENTIFIER", "instrument"),
240 ("EQ", "="),
241 ("STRING_LITERAL", "HSC"),
242 ("AND", "AND"),
243 ("SIMPLE_IDENTIFIER", "detector"),
244 ("NE", "!="),
245 ("NUMERIC_LITERAL", "9"),
246 ("RPAREN", ")"),
247 ("OR", "OR"),
248 ("SIMPLE_IDENTIFIER", "instrument"),
249 ("EQ", "="),
250 ("STRING_LITERAL", "CFHT"),
251 ("RPAREN", ")"),
252 ("AND", "AND"),
253 ("SIMPLE_IDENTIFIER", "tract"),
254 ("EQ", "="),
255 ("NUMERIC_LITERAL", "8766"),
256 ("AND", "AND"),
257 ("QUALIFIED_IDENTIFIER", "patch.cell_x"),
258 ("GT", ">"),
259 ("NUMERIC_LITERAL", "5"),
260 ("AND", "AND"),
261 ("QUALIFIED_IDENTIFIER", "patch.cell_y"),
262 ("LT", "<"),
263 ("NUMERIC_LITERAL", "4"),
264 ("AND", "AND"),
265 ("SIMPLE_IDENTIFIER", "band"),
266 ("EQ", "="),
267 ("STRING_LITERAL", "i"),
268 ("OR", "OR"),
269 ("SIMPLE_IDENTIFIER", "visit"),
270 ("IN", "IN"),
271 ("LPAREN", "("),
272 ("RANGE_LITERAL", (1, 50, 2)),
273 ("RPAREN", ")"),
274 )
275 lexer.input(expr)
276 for type, value in tokens:
277 self._assertToken(lexer.token(), type, value)
278 self.assertIsNone(lexer.token())
280 def testExceptions(self):
281 """Test for exception contents"""
283 def _assertExc(exc, expr, remain, pos, lineno):
284 """Check exception attribute values"""
285 self.assertEqual(exc.expression, expr)
286 self.assertEqual(exc.remain, remain)
287 self.assertEqual(exc.pos, pos)
288 self.assertEqual(exc.lineno, lineno)
290 lexer = ParserLex.make_lexer()
291 expr = "a.b.c.d"
292 lexer.input(expr)
293 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c")
294 with self.assertRaises(ParserLexError) as catcher:
295 lexer.token()
296 _assertExc(catcher.exception, expr, ".d", 5, 1)
298 lexer = ParserLex.make_lexer()
299 expr = "a \n& b"
300 lexer.input(expr)
301 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a")
302 with self.assertRaises(ParserLexError) as catcher:
303 lexer.token()
304 _assertExc(catcher.exception, expr, "& b", 3, 2)
306 lexer = ParserLex.make_lexer()
307 expr = "a\n=\n1e5.e2"
308 lexer.input(expr)
309 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a")
310 self._assertToken(lexer.token(), "EQ", "=")
311 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e5")
312 with self.assertRaises(ParserLexError) as catcher:
313 lexer.token()
314 _assertExc(catcher.exception, expr, ".e2", 7, 3)
316 # zero stride in range literal
317 lexer = ParserLex.make_lexer()
318 expr = "1..2:0"
319 lexer.input(expr)
320 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None))
321 with self.assertRaises(ParserLexError) as catcher:
322 lexer.token()
323 _assertExc(catcher.exception, expr, ":0", 4, 1)
325 # negative stride in range literal
326 lexer = ParserLex.make_lexer()
327 expr = "1..2:-10"
328 lexer.input(expr)
329 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None))
330 with self.assertRaises(ParserLexError) as catcher:
331 lexer.token()
332 _assertExc(catcher.exception, expr, ":-10", 4, 1)
335if __name__ == "__main__":
336 unittest.main()