Coverage for tests/test_exprParserLex.py: 9%
180 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 03:16 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 03:16 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <https://www.gnu.org/licenses/>.
28"""Simple unit test for expr_parser/parserLex module.
29"""
31import re
32import unittest
34from lsst.daf.butler.registry.queries.expressions.parser import ParserLex, ParserLexError
37class ParserLexTestCase(unittest.TestCase):
38 """A test case for ParserLex"""
40 def _assertToken(self, token, type, value, lineno=None, lexpos=None):
41 self.assertIsNotNone(token)
42 self.assertEqual(token.type, type)
43 self.assertEqual(token.value, value)
44 if lineno is not None:
45 self.assertEqual(token.lineno, lineno)
46 if lexpos is not None:
47 self.assertEqual(token.lexpos, lexpos)
49 def setUp(self):
50 pass
52 def tearDown(self):
53 pass
55 def testInstantiate(self):
56 """Tests for making ParserLex instances"""
57 default_reflags = re.IGNORECASE | re.VERBOSE
58 lexer = ParserLex.make_lexer()
59 self.assertEqual(lexer.lexreflags, default_reflags)
61 lexer = ParserLex.make_lexer(reflags=re.DOTALL)
62 self.assertEqual(lexer.lexreflags, re.DOTALL | default_reflags)
64 def testSimpleTokens(self):
65 """Test for simple tokens"""
66 lexer = ParserLex.make_lexer()
68 lexer.input("=!= <<= >>= +-*/()")
69 self._assertToken(lexer.token(), "EQ", "=")
70 self._assertToken(lexer.token(), "NE", "!=")
71 self._assertToken(lexer.token(), "LT", "<")
72 self._assertToken(lexer.token(), "LE", "<=")
73 self._assertToken(lexer.token(), "GT", ">")
74 self._assertToken(lexer.token(), "GE", ">=")
75 self._assertToken(lexer.token(), "ADD", "+")
76 self._assertToken(lexer.token(), "SUB", "-")
77 self._assertToken(lexer.token(), "MUL", "*")
78 self._assertToken(lexer.token(), "DIV", "/")
79 self._assertToken(lexer.token(), "LPAREN", "(")
80 self._assertToken(lexer.token(), "RPAREN", ")")
81 self.assertIsNone(lexer.token())
83 def testReservedTokens(self):
84 """Test for reserved words"""
85 lexer = ParserLex.make_lexer()
87 tokens = "NOT IN OR AND OVERLAPS"
88 lexer.input(tokens)
89 for token in tokens.split():
90 self._assertToken(lexer.token(), token, token)
91 self.assertIsNone(lexer.token())
93 tokens = "not in or and overlaps"
94 lexer.input(tokens)
95 for token in tokens.split():
96 self._assertToken(lexer.token(), token.upper(), token.upper())
97 self.assertIsNone(lexer.token())
99 # not reserved
100 token = "NOTIN"
101 lexer.input(token)
102 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", token)
103 self.assertIsNone(lexer.token())
105 def testStringLiteral(self):
106 """Test for string literals"""
107 lexer = ParserLex.make_lexer()
109 lexer.input("''")
110 self._assertToken(lexer.token(), "STRING_LITERAL", "")
111 self.assertIsNone(lexer.token())
113 lexer.input("'string'")
114 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
115 self.assertIsNone(lexer.token())
117 lexer.input("'string' 'string'\n'string'")
118 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
119 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
120 self._assertToken(lexer.token(), "STRING_LITERAL", "string")
121 self.assertIsNone(lexer.token())
123 # odd newline inside string
124 lexer.input("'string\nstring'")
125 with self.assertRaises(ParserLexError):
126 lexer.token()
128 lexer.input("'string")
129 with self.assertRaises(ParserLexError):
130 lexer.token()
132 def testNumericLiteral(self):
133 """Test for numeric literals"""
134 lexer = ParserLex.make_lexer()
136 lexer.input("0 100 999. 100.1 1e10 1e-10 1.e+20 .2E5")
137 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "0")
138 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100")
139 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "999.")
140 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100.1")
141 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e10")
142 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e-10")
143 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1.e+20")
144 self._assertToken(lexer.token(), "NUMERIC_LITERAL", ".2E5")
145 self.assertIsNone(lexer.token())
147 def testRangeLiteral(self):
148 """Test for range literals"""
149 lexer = ParserLex.make_lexer()
151 lexer.input("0..10 -10..-1 -10..10:2 0 .. 10 0 .. 10 : 2 ")
152 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None))
153 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, -1, None))
154 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, 10, 2))
155 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None))
156 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, 2))
157 self.assertIsNone(lexer.token())
159 def testTimeLiteral(self):
160 """Test for time literals"""
161 lexer = ParserLex.make_lexer()
163 # string can contain anything, lexer does not check it
164 lexer.input("T'2020-03-30' T'2020-03-30 00:00:00' T'2020-03-30T00:00:00' T'123.456' T'time'")
165 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30")
166 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30 00:00:00")
167 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30T00:00:00")
168 self._assertToken(lexer.token(), "TIME_LITERAL", "123.456")
169 self._assertToken(lexer.token(), "TIME_LITERAL", "time")
170 self.assertIsNone(lexer.token())
172 def testIdentifier(self):
173 """Test for numeric literals"""
174 lexer = ParserLex.make_lexer()
176 lexer.input("ID id _012 a_b_C")
177 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "ID")
178 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id")
179 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "_012")
180 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a_b_C")
181 self.assertIsNone(lexer.token())
183 lexer.input("a.b a.b.c _._ _._._")
184 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b")
185 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c")
186 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._")
187 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._._")
188 self.assertIsNone(lexer.token())
190 lexer.input(".id")
191 with self.assertRaises(ParserLexError):
192 lexer.token()
194 lexer.input("id.")
195 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id")
196 with self.assertRaises(ParserLexError):
197 lexer.token()
199 lexer.input("id.id.id.id")
200 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "id.id.id")
201 with self.assertRaises(ParserLexError):
202 lexer.token()
204 def testExpression(self):
205 """Test for more or less complete expression"""
206 lexer = ParserLex.make_lexer()
208 expr = (
209 "((instrument='HSC' AND detector != 9) OR instrument='CFHT') "
210 "AND tract=8766 AND patch.cell_x > 5 AND "
211 "patch.cell_y < 4 AND band='i' "
212 "or visit IN (1..50:2)"
213 )
214 tokens = (
215 ("LPAREN", "("),
216 ("LPAREN", "("),
217 ("SIMPLE_IDENTIFIER", "instrument"),
218 ("EQ", "="),
219 ("STRING_LITERAL", "HSC"),
220 ("AND", "AND"),
221 ("SIMPLE_IDENTIFIER", "detector"),
222 ("NE", "!="),
223 ("NUMERIC_LITERAL", "9"),
224 ("RPAREN", ")"),
225 ("OR", "OR"),
226 ("SIMPLE_IDENTIFIER", "instrument"),
227 ("EQ", "="),
228 ("STRING_LITERAL", "CFHT"),
229 ("RPAREN", ")"),
230 ("AND", "AND"),
231 ("SIMPLE_IDENTIFIER", "tract"),
232 ("EQ", "="),
233 ("NUMERIC_LITERAL", "8766"),
234 ("AND", "AND"),
235 ("QUALIFIED_IDENTIFIER", "patch.cell_x"),
236 ("GT", ">"),
237 ("NUMERIC_LITERAL", "5"),
238 ("AND", "AND"),
239 ("QUALIFIED_IDENTIFIER", "patch.cell_y"),
240 ("LT", "<"),
241 ("NUMERIC_LITERAL", "4"),
242 ("AND", "AND"),
243 ("SIMPLE_IDENTIFIER", "band"),
244 ("EQ", "="),
245 ("STRING_LITERAL", "i"),
246 ("OR", "OR"),
247 ("SIMPLE_IDENTIFIER", "visit"),
248 ("IN", "IN"),
249 ("LPAREN", "("),
250 ("RANGE_LITERAL", (1, 50, 2)),
251 ("RPAREN", ")"),
252 )
253 lexer.input(expr)
254 for type, value in tokens:
255 self._assertToken(lexer.token(), type, value)
256 self.assertIsNone(lexer.token())
258 def testExceptions(self):
259 """Test for exception contents"""
261 def _assertExc(exc, expr, remain, pos, lineno):
262 """Check exception attribute values"""
263 self.assertEqual(exc.expression, expr)
264 self.assertEqual(exc.remain, remain)
265 self.assertEqual(exc.pos, pos)
266 self.assertEqual(exc.lineno, lineno)
268 lexer = ParserLex.make_lexer()
269 expr = "a.b.c.d"
270 lexer.input(expr)
271 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c")
272 with self.assertRaises(ParserLexError) as catcher:
273 lexer.token()
274 _assertExc(catcher.exception, expr, ".d", 5, 1)
276 lexer = ParserLex.make_lexer()
277 expr = "a \n& b"
278 lexer.input(expr)
279 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a")
280 with self.assertRaises(ParserLexError) as catcher:
281 lexer.token()
282 _assertExc(catcher.exception, expr, "& b", 3, 2)
284 lexer = ParserLex.make_lexer()
285 expr = "a\n=\n1e5.e2"
286 lexer.input(expr)
287 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a")
288 self._assertToken(lexer.token(), "EQ", "=")
289 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e5")
290 with self.assertRaises(ParserLexError) as catcher:
291 lexer.token()
292 _assertExc(catcher.exception, expr, ".e2", 7, 3)
294 # zero stride in range literal
295 lexer = ParserLex.make_lexer()
296 expr = "1..2:0"
297 lexer.input(expr)
298 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None))
299 with self.assertRaises(ParserLexError) as catcher:
300 lexer.token()
301 _assertExc(catcher.exception, expr, ":0", 4, 1)
303 # negative stride in range literal
304 lexer = ParserLex.make_lexer()
305 expr = "1..2:-10"
306 lexer.input(expr)
307 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None))
308 with self.assertRaises(ParserLexError) as catcher:
309 lexer.token()
310 _assertExc(catcher.exception, expr, ":-10", 4, 1)
313if __name__ == "__main__":
314 unittest.main()