Coverage for tests/test_exprParserLex.py: 9%
180 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-04 02:06 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-04 02:06 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for expr_parser/parserLex module.
23"""
25import re
26import unittest
28from lsst.daf.butler.registry.queries.expressions.parser import ParserLex, ParserLexError
31class ParserLexTestCase(unittest.TestCase):
32 """A test case for ParserLex"""
34 def _assertToken(self, token, type, value, lineno=None, lexpos=None):
35 self.assertIsNotNone(token)
36 self.assertEqual(token.type, type)
37 self.assertEqual(token.value, value)
38 if lineno is not None:
39 self.assertEqual(token.lineno, lineno)
40 if lexpos is not None:
41 self.assertEqual(token.lexpos, lexpos)
43 def setUp(self):
44 pass
46 def tearDown(self):
47 pass
49 def testInstantiate(self):
50 """Tests for making ParserLex instances"""
52 default_reflags = re.IGNORECASE | re.VERBOSE
53 lexer = ParserLex.make_lexer()
54 self.assertEqual(lexer.lexreflags, default_reflags)
56 lexer = ParserLex.make_lexer(reflags=re.DOTALL)
57 self.assertEqual(lexer.lexreflags, re.DOTALL | default_reflags)
59 def testSimpleTokens(self):
60 """Test for simple tokens"""
61 lexer = ParserLex.make_lexer()
63 lexer.input("=!= <<= >>= +-*/()")
64 self._assertToken(lexer.token(), "EQ", "=")
65 self._assertToken(lexer.token(), "NE", "!=")
66 self._assertToken(lexer.token(), "LT", "<")
67 self._assertToken(lexer.token(), "LE", "<=")
68 self._assertToken(lexer.token(), "GT", ">")
69 self._assertToken(lexer.token(), "GE", ">=")
70 self._assertToken(lexer.token(), "ADD", "+")
71 self._assertToken(lexer.token(), "SUB", "-")
72 self._assertToken(lexer.token(), "MUL", "*")
73 self._assertToken(lexer.token(), "DIV", "/")
74 self._assertToken(lexer.token(), "LPAREN", "(")
75 self._assertToken(lexer.token(), "RPAREN", ")")
76 self.assertIsNone(lexer.token())
78 def testReservedTokens(self):
79 """Test for reserved words"""
80 lexer = ParserLex.make_lexer()
82 tokens = "NOT IN OR AND OVERLAPS"
83 lexer.input(tokens)
84 for token in tokens.split():
85 self._assertToken(lexer.token(), token, token)
86 self.assertIsNone(lexer.token())
88 tokens = "not in or and overlaps"
89 lexer.input(tokens)
90 for token in tokens.split():
91 self._assertToken(lexer.token(), token.upper(), token.upper())
92 self.assertIsNone(lexer.token())
94 # not reserved
95 token = "NOTIN"
96 lexer.input(token)
97 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", token)
98 self.assertIsNone(lexer.token())
100 def testStringLiteral(self):
101 """Test for string literals"""
102 lexer = ParserLex.make_lexer()
104 lexer.input("''")
105 self._assertToken(lexer.token(), "STRING_LITERAL", "")
106 self.assertIsNone(lexer.token())
108 lexer.input("'string'")
109 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
110 self.assertIsNone(lexer.token())
112 lexer.input("'string' 'string'\n'string'")
113 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
114 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
115 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
116 self.assertIsNone(lexer.token())
118 # odd newline inside string
119 lexer.input("'string\nstring'")
120 with self.assertRaises(ParserLexError):
121 lexer.token()
123 lexer.input("'string")
124 with self.assertRaises(ParserLexError):
125 lexer.token()
127 def testNumericLiteral(self):
128 """Test for numeric literals"""
129 lexer = ParserLex.make_lexer()
131 lexer.input("0 100 999. 100.1 1e10 1e-10 1.e+20 .2E5")
132 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "0")
133 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100")
134 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "999.")
135 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100.1")
136 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e10")
137 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e-10")
138 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1.e+20")
139 self._assertToken(lexer.token(), "NUMERIC_LITERAL", ".2E5")
140 self.assertIsNone(lexer.token())
142 def testRangeLiteral(self):
143 """Test for range literals"""
144 lexer = ParserLex.make_lexer()
146 lexer.input("0..10 -10..-1 -10..10:2 0 .. 10 0 .. 10 : 2 ")
147 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None))
148 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, -1, None))
149 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, 10, 2))
150 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None))
151 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, 2))
152 self.assertIsNone(lexer.token())
154 def testTimeLiteral(self):
155 """Test for time literals"""
156 lexer = ParserLex.make_lexer()
158 # string can contain anything, lexer does not check it
159 lexer.input("T'2020-03-30' T'2020-03-30 00:00:00' T'2020-03-30T00:00:00' T'123.456' T'time'")
160 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30")
161 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30 00:00:00")
162 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30T00:00:00")
163 self._assertToken(lexer.token(), "TIME_LITERAL", "123.456")
164 self._assertToken(lexer.token(), "TIME_LITERAL", "time")
165 self.assertIsNone(lexer.token())
167 def testIdentifier(self):
168 """Test for numeric literals"""
169 lexer = ParserLex.make_lexer()
171 lexer.input("ID id _012 a_b_C")
172 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "ID")
173 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id")
174 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "_012")
175 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a_b_C")
176 self.assertIsNone(lexer.token())
178 lexer.input("a.b a.b.c _._ _._._")
179 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b")
180 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c")
181 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._")
182 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._._")
183 self.assertIsNone(lexer.token())
185 lexer.input(".id")
186 with self.assertRaises(ParserLexError):
187 lexer.token()
189 lexer.input("id.")
190 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id")
191 with self.assertRaises(ParserLexError):
192 lexer.token()
194 lexer.input("id.id.id.id")
195 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "id.id.id")
196 with self.assertRaises(ParserLexError):
197 lexer.token()
199 def testExpression(self):
200 """Test for more or less complete expression"""
201 lexer = ParserLex.make_lexer()
203 expr = (
204 "((instrument='HSC' AND detector != 9) OR instrument='CFHT') "
205 "AND tract=8766 AND patch.cell_x > 5 AND "
206 "patch.cell_y < 4 AND band='i' "
207 "or visit IN (1..50:2)"
208 )
209 tokens = (
210 ("LPAREN", "("),
211 ("LPAREN", "("),
212 ("SIMPLE_IDENTIFIER", "instrument"),
213 ("EQ", "="),
214 ("STRING_LITERAL", "HSC"),
215 ("AND", "AND"),
216 ("SIMPLE_IDENTIFIER", "detector"),
217 ("NE", "!="),
218 ("NUMERIC_LITERAL", "9"),
219 ("RPAREN", ")"),
220 ("OR", "OR"),
221 ("SIMPLE_IDENTIFIER", "instrument"),
222 ("EQ", "="),
223 ("STRING_LITERAL", "CFHT"),
224 ("RPAREN", ")"),
225 ("AND", "AND"),
226 ("SIMPLE_IDENTIFIER", "tract"),
227 ("EQ", "="),
228 ("NUMERIC_LITERAL", "8766"),
229 ("AND", "AND"),
230 ("QUALIFIED_IDENTIFIER", "patch.cell_x"),
231 ("GT", ">"),
232 ("NUMERIC_LITERAL", "5"),
233 ("AND", "AND"),
234 ("QUALIFIED_IDENTIFIER", "patch.cell_y"),
235 ("LT", "<"),
236 ("NUMERIC_LITERAL", "4"),
237 ("AND", "AND"),
238 ("SIMPLE_IDENTIFIER", "band"),
239 ("EQ", "="),
240 ("STRING_LITERAL", "i"),
241 ("OR", "OR"),
242 ("SIMPLE_IDENTIFIER", "visit"),
243 ("IN", "IN"),
244 ("LPAREN", "("),
245 ("RANGE_LITERAL", (1, 50, 2)),
246 ("RPAREN", ")"),
247 )
248 lexer.input(expr)
249 for type, value in tokens:
250 self._assertToken(lexer.token(), type, value)
251 self.assertIsNone(lexer.token())
253 def testExceptions(self):
254 """Test for exception contents"""
256 def _assertExc(exc, expr, remain, pos, lineno):
257 """Check exception attribute values"""
258 self.assertEqual(exc.expression, expr)
259 self.assertEqual(exc.remain, remain)
260 self.assertEqual(exc.pos, pos)
261 self.assertEqual(exc.lineno, lineno)
263 lexer = ParserLex.make_lexer()
264 expr = "a.b.c.d"
265 lexer.input(expr)
266 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c")
267 with self.assertRaises(ParserLexError) as catcher:
268 lexer.token()
269 _assertExc(catcher.exception, expr, ".d", 5, 1)
271 lexer = ParserLex.make_lexer()
272 expr = "a \n& b"
273 lexer.input(expr)
274 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a")
275 with self.assertRaises(ParserLexError) as catcher:
276 lexer.token()
277 _assertExc(catcher.exception, expr, "& b", 3, 2)
279 lexer = ParserLex.make_lexer()
280 expr = "a\n=\n1e5.e2"
281 lexer.input(expr)
282 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a")
283 self._assertToken(lexer.token(), "EQ", "=")
284 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e5")
285 with self.assertRaises(ParserLexError) as catcher:
286 lexer.token()
287 _assertExc(catcher.exception, expr, ".e2", 7, 3)
289 # zero stride in range literal
290 lexer = ParserLex.make_lexer()
291 expr = "1..2:0"
292 lexer.input(expr)
293 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None))
294 with self.assertRaises(ParserLexError) as catcher:
295 lexer.token()
296 _assertExc(catcher.exception, expr, ":0", 4, 1)
298 # negative stride in range literal
299 lexer = ParserLex.make_lexer()
300 expr = "1..2:-10"
301 lexer.input(expr)
302 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None))
303 with self.assertRaises(ParserLexError) as catcher:
304 lexer.token()
305 _assertExc(catcher.exception, expr, ":-10", 4, 1)
308if __name__ == "__main__":
309 unittest.main()