Coverage for tests/test_exprParserLex.py: 9%

180 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-13 10:57 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <https://www.gnu.org/licenses/>. 

27 

28"""Simple unit test for expr_parser/parserLex module. 

29""" 

30 

31import re 

32import unittest 

33 

34from lsst.daf.butler.registry.queries.expressions.parser import ParserLex, ParserLexError 

35 

36 

37class ParserLexTestCase(unittest.TestCase): 

38 """A test case for ParserLex""" 

39 

40 def _assertToken(self, token, type, value, lineno=None, lexpos=None): 

41 self.assertIsNotNone(token) 

42 self.assertEqual(token.type, type) 

43 self.assertEqual(token.value, value) 

44 if lineno is not None: 

45 self.assertEqual(token.lineno, lineno) 

46 if lexpos is not None: 

47 self.assertEqual(token.lexpos, lexpos) 

48 

49 def setUp(self): 

50 pass 

51 

52 def tearDown(self): 

53 pass 

54 

55 def testInstantiate(self): 

56 """Tests for making ParserLex instances""" 

57 default_reflags = re.IGNORECASE | re.VERBOSE 

58 lexer = ParserLex.make_lexer() 

59 self.assertEqual(lexer.lexreflags, default_reflags) 

60 

61 lexer = ParserLex.make_lexer(reflags=re.DOTALL) 

62 self.assertEqual(lexer.lexreflags, re.DOTALL | default_reflags) 

63 

64 def testSimpleTokens(self): 

65 """Test for simple tokens""" 

66 lexer = ParserLex.make_lexer() 

67 

68 lexer.input("=!= <<= >>= +-*/()") 

69 self._assertToken(lexer.token(), "EQ", "=") 

70 self._assertToken(lexer.token(), "NE", "!=") 

71 self._assertToken(lexer.token(), "LT", "<") 

72 self._assertToken(lexer.token(), "LE", "<=") 

73 self._assertToken(lexer.token(), "GT", ">") 

74 self._assertToken(lexer.token(), "GE", ">=") 

75 self._assertToken(lexer.token(), "ADD", "+") 

76 self._assertToken(lexer.token(), "SUB", "-") 

77 self._assertToken(lexer.token(), "MUL", "*") 

78 self._assertToken(lexer.token(), "DIV", "/") 

79 self._assertToken(lexer.token(), "LPAREN", "(") 

80 self._assertToken(lexer.token(), "RPAREN", ")") 

81 self.assertIsNone(lexer.token()) 

82 

83 def testReservedTokens(self): 

84 """Test for reserved words""" 

85 lexer = ParserLex.make_lexer() 

86 

87 tokens = "NOT IN OR AND OVERLAPS" 

88 lexer.input(tokens) 

89 for token in tokens.split(): 

90 self._assertToken(lexer.token(), token, token) 

91 self.assertIsNone(lexer.token()) 

92 

93 tokens = "not in or and overlaps" 

94 lexer.input(tokens) 

95 for token in tokens.split(): 

96 self._assertToken(lexer.token(), token.upper(), token.upper()) 

97 self.assertIsNone(lexer.token()) 

98 

99 # not reserved 

100 token = "NOTIN" 

101 lexer.input(token) 

102 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", token) 

103 self.assertIsNone(lexer.token()) 

104 

105 def testStringLiteral(self): 

106 """Test for string literals""" 

107 lexer = ParserLex.make_lexer() 

108 

109 lexer.input("''") 

110 self._assertToken(lexer.token(), "STRING_LITERAL", "") 

111 self.assertIsNone(lexer.token()) 

112 

113 lexer.input("'string'") 

114 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

115 self.assertIsNone(lexer.token()) 

116 

117 lexer.input("'string' 'string'\n'string'") 

118 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

119 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

120 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

121 self.assertIsNone(lexer.token()) 

122 

123 # odd newline inside string 

124 lexer.input("'string\nstring'") 

125 with self.assertRaises(ParserLexError): 

126 lexer.token() 

127 

128 lexer.input("'string") 

129 with self.assertRaises(ParserLexError): 

130 lexer.token() 

131 

132 def testNumericLiteral(self): 

133 """Test for numeric literals""" 

134 lexer = ParserLex.make_lexer() 

135 

136 lexer.input("0 100 999. 100.1 1e10 1e-10 1.e+20 .2E5") 

137 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "0") 

138 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100") 

139 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "999.") 

140 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100.1") 

141 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e10") 

142 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e-10") 

143 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1.e+20") 

144 self._assertToken(lexer.token(), "NUMERIC_LITERAL", ".2E5") 

145 self.assertIsNone(lexer.token()) 

146 

147 def testRangeLiteral(self): 

148 """Test for range literals""" 

149 lexer = ParserLex.make_lexer() 

150 

151 lexer.input("0..10 -10..-1 -10..10:2 0 .. 10 0 .. 10 : 2 ") 

152 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None)) 

153 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, -1, None)) 

154 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, 10, 2)) 

155 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None)) 

156 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, 2)) 

157 self.assertIsNone(lexer.token()) 

158 

159 def testTimeLiteral(self): 

160 """Test for time literals""" 

161 lexer = ParserLex.make_lexer() 

162 

163 # string can contain anything, lexer does not check it 

164 lexer.input("T'2020-03-30' T'2020-03-30 00:00:00' T'2020-03-30T00:00:00' T'123.456' T'time'") 

165 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30") 

166 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30 00:00:00") 

167 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30T00:00:00") 

168 self._assertToken(lexer.token(), "TIME_LITERAL", "123.456") 

169 self._assertToken(lexer.token(), "TIME_LITERAL", "time") 

170 self.assertIsNone(lexer.token()) 

171 

172 def testIdentifier(self): 

173 """Test for numeric literals""" 

174 lexer = ParserLex.make_lexer() 

175 

176 lexer.input("ID id _012 a_b_C") 

177 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "ID") 

178 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id") 

179 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "_012") 

180 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a_b_C") 

181 self.assertIsNone(lexer.token()) 

182 

183 lexer.input("a.b a.b.c _._ _._._") 

184 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b") 

185 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c") 

186 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._") 

187 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._._") 

188 self.assertIsNone(lexer.token()) 

189 

190 lexer.input(".id") 

191 with self.assertRaises(ParserLexError): 

192 lexer.token() 

193 

194 lexer.input("id.") 

195 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id") 

196 with self.assertRaises(ParserLexError): 

197 lexer.token() 

198 

199 lexer.input("id.id.id.id") 

200 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "id.id.id") 

201 with self.assertRaises(ParserLexError): 

202 lexer.token() 

203 

204 def testExpression(self): 

205 """Test for more or less complete expression""" 

206 lexer = ParserLex.make_lexer() 

207 

208 expr = ( 

209 "((instrument='HSC' AND detector != 9) OR instrument='CFHT') " 

210 "AND tract=8766 AND patch.cell_x > 5 AND " 

211 "patch.cell_y < 4 AND band='i' " 

212 "or visit IN (1..50:2)" 

213 ) 

214 tokens = ( 

215 ("LPAREN", "("), 

216 ("LPAREN", "("), 

217 ("SIMPLE_IDENTIFIER", "instrument"), 

218 ("EQ", "="), 

219 ("STRING_LITERAL", "HSC"), 

220 ("AND", "AND"), 

221 ("SIMPLE_IDENTIFIER", "detector"), 

222 ("NE", "!="), 

223 ("NUMERIC_LITERAL", "9"), 

224 ("RPAREN", ")"), 

225 ("OR", "OR"), 

226 ("SIMPLE_IDENTIFIER", "instrument"), 

227 ("EQ", "="), 

228 ("STRING_LITERAL", "CFHT"), 

229 ("RPAREN", ")"), 

230 ("AND", "AND"), 

231 ("SIMPLE_IDENTIFIER", "tract"), 

232 ("EQ", "="), 

233 ("NUMERIC_LITERAL", "8766"), 

234 ("AND", "AND"), 

235 ("QUALIFIED_IDENTIFIER", "patch.cell_x"), 

236 ("GT", ">"), 

237 ("NUMERIC_LITERAL", "5"), 

238 ("AND", "AND"), 

239 ("QUALIFIED_IDENTIFIER", "patch.cell_y"), 

240 ("LT", "<"), 

241 ("NUMERIC_LITERAL", "4"), 

242 ("AND", "AND"), 

243 ("SIMPLE_IDENTIFIER", "band"), 

244 ("EQ", "="), 

245 ("STRING_LITERAL", "i"), 

246 ("OR", "OR"), 

247 ("SIMPLE_IDENTIFIER", "visit"), 

248 ("IN", "IN"), 

249 ("LPAREN", "("), 

250 ("RANGE_LITERAL", (1, 50, 2)), 

251 ("RPAREN", ")"), 

252 ) 

253 lexer.input(expr) 

254 for type, value in tokens: 

255 self._assertToken(lexer.token(), type, value) 

256 self.assertIsNone(lexer.token()) 

257 

258 def testExceptions(self): 

259 """Test for exception contents""" 

260 

261 def _assertExc(exc, expr, remain, pos, lineno): 

262 """Check exception attribute values""" 

263 self.assertEqual(exc.expression, expr) 

264 self.assertEqual(exc.remain, remain) 

265 self.assertEqual(exc.pos, pos) 

266 self.assertEqual(exc.lineno, lineno) 

267 

268 lexer = ParserLex.make_lexer() 

269 expr = "a.b.c.d" 

270 lexer.input(expr) 

271 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c") 

272 with self.assertRaises(ParserLexError) as catcher: 

273 lexer.token() 

274 _assertExc(catcher.exception, expr, ".d", 5, 1) 

275 

276 lexer = ParserLex.make_lexer() 

277 expr = "a \n& b" 

278 lexer.input(expr) 

279 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a") 

280 with self.assertRaises(ParserLexError) as catcher: 

281 lexer.token() 

282 _assertExc(catcher.exception, expr, "& b", 3, 2) 

283 

284 lexer = ParserLex.make_lexer() 

285 expr = "a\n=\n1e5.e2" 

286 lexer.input(expr) 

287 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a") 

288 self._assertToken(lexer.token(), "EQ", "=") 

289 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e5") 

290 with self.assertRaises(ParserLexError) as catcher: 

291 lexer.token() 

292 _assertExc(catcher.exception, expr, ".e2", 7, 3) 

293 

294 # zero stride in range literal 

295 lexer = ParserLex.make_lexer() 

296 expr = "1..2:0" 

297 lexer.input(expr) 

298 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None)) 

299 with self.assertRaises(ParserLexError) as catcher: 

300 lexer.token() 

301 _assertExc(catcher.exception, expr, ":0", 4, 1) 

302 

303 # negative stride in range literal 

304 lexer = ParserLex.make_lexer() 

305 expr = "1..2:-10" 

306 lexer.input(expr) 

307 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None)) 

308 with self.assertRaises(ParserLexError) as catcher: 

309 lexer.token() 

310 _assertExc(catcher.exception, expr, ":-10", 4, 1) 

311 

312 

313if __name__ == "__main__": 

314 unittest.main()