Coverage for tests/test_exprParserLex.py: 9%

180 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-02 09:50 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for expr_parser/parserLex module. 

23""" 

24 

25import re 

26import unittest 

27 

28from lsst.daf.butler.registry.queries.expressions.parser import ParserLex, ParserLexError 

29 

30 

31class ParserLexTestCase(unittest.TestCase): 

32 """A test case for ParserLex""" 

33 

34 def _assertToken(self, token, type, value, lineno=None, lexpos=None): 

35 self.assertIsNotNone(token) 

36 self.assertEqual(token.type, type) 

37 self.assertEqual(token.value, value) 

38 if lineno is not None: 

39 self.assertEqual(token.lineno, lineno) 

40 if lexpos is not None: 

41 self.assertEqual(token.lexpos, lexpos) 

42 

43 def setUp(self): 

44 pass 

45 

46 def tearDown(self): 

47 pass 

48 

49 def testInstantiate(self): 

50 """Tests for making ParserLex instances""" 

51 

52 default_reflags = re.IGNORECASE | re.VERBOSE 

53 lexer = ParserLex.make_lexer() 

54 self.assertEqual(lexer.lexreflags, default_reflags) 

55 

56 lexer = ParserLex.make_lexer(reflags=re.DOTALL) 

57 self.assertEqual(lexer.lexreflags, re.DOTALL | default_reflags) 

58 

59 def testSimpleTokens(self): 

60 """Test for simple tokens""" 

61 lexer = ParserLex.make_lexer() 

62 

63 lexer.input("=!= <<= >>= +-*/()") 

64 self._assertToken(lexer.token(), "EQ", "=") 

65 self._assertToken(lexer.token(), "NE", "!=") 

66 self._assertToken(lexer.token(), "LT", "<") 

67 self._assertToken(lexer.token(), "LE", "<=") 

68 self._assertToken(lexer.token(), "GT", ">") 

69 self._assertToken(lexer.token(), "GE", ">=") 

70 self._assertToken(lexer.token(), "ADD", "+") 

71 self._assertToken(lexer.token(), "SUB", "-") 

72 self._assertToken(lexer.token(), "MUL", "*") 

73 self._assertToken(lexer.token(), "DIV", "/") 

74 self._assertToken(lexer.token(), "LPAREN", "(") 

75 self._assertToken(lexer.token(), "RPAREN", ")") 

76 self.assertIsNone(lexer.token()) 

77 

78 def testReservedTokens(self): 

79 """Test for reserved words""" 

80 lexer = ParserLex.make_lexer() 

81 

82 tokens = "NOT IN OR AND OVERLAPS" 

83 lexer.input(tokens) 

84 for token in tokens.split(): 

85 self._assertToken(lexer.token(), token, token) 

86 self.assertIsNone(lexer.token()) 

87 

88 tokens = "not in or and overlaps" 

89 lexer.input(tokens) 

90 for token in tokens.split(): 

91 self._assertToken(lexer.token(), token.upper(), token.upper()) 

92 self.assertIsNone(lexer.token()) 

93 

94 # not reserved 

95 token = "NOTIN" 

96 lexer.input(token) 

97 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", token) 

98 self.assertIsNone(lexer.token()) 

99 

100 def testStringLiteral(self): 

101 """Test for string literals""" 

102 lexer = ParserLex.make_lexer() 

103 

104 lexer.input("''") 

105 self._assertToken(lexer.token(), "STRING_LITERAL", "") 

106 self.assertIsNone(lexer.token()) 

107 

108 lexer.input("'string'") 

109 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

110 self.assertIsNone(lexer.token()) 

111 

112 lexer.input("'string' 'string'\n'string'") 

113 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

114 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

115 self._assertToken(lexer.token(), "STRING_LITERAL", "string") 

116 self.assertIsNone(lexer.token()) 

117 

118 # odd newline inside string 

119 lexer.input("'string\nstring'") 

120 with self.assertRaises(ParserLexError): 

121 lexer.token() 

122 

123 lexer.input("'string") 

124 with self.assertRaises(ParserLexError): 

125 lexer.token() 

126 

127 def testNumericLiteral(self): 

128 """Test for numeric literals""" 

129 lexer = ParserLex.make_lexer() 

130 

131 lexer.input("0 100 999. 100.1 1e10 1e-10 1.e+20 .2E5") 

132 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "0") 

133 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100") 

134 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "999.") 

135 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "100.1") 

136 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e10") 

137 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e-10") 

138 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1.e+20") 

139 self._assertToken(lexer.token(), "NUMERIC_LITERAL", ".2E5") 

140 self.assertIsNone(lexer.token()) 

141 

142 def testRangeLiteral(self): 

143 """Test for range literals""" 

144 lexer = ParserLex.make_lexer() 

145 

146 lexer.input("0..10 -10..-1 -10..10:2 0 .. 10 0 .. 10 : 2 ") 

147 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None)) 

148 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, -1, None)) 

149 self._assertToken(lexer.token(), "RANGE_LITERAL", (-10, 10, 2)) 

150 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, None)) 

151 self._assertToken(lexer.token(), "RANGE_LITERAL", (0, 10, 2)) 

152 self.assertIsNone(lexer.token()) 

153 

154 def testTimeLiteral(self): 

155 """Test for time literals""" 

156 lexer = ParserLex.make_lexer() 

157 

158 # string can contain anything, lexer does not check it 

159 lexer.input("T'2020-03-30' T'2020-03-30 00:00:00' T'2020-03-30T00:00:00' T'123.456' T'time'") 

160 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30") 

161 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30 00:00:00") 

162 self._assertToken(lexer.token(), "TIME_LITERAL", "2020-03-30T00:00:00") 

163 self._assertToken(lexer.token(), "TIME_LITERAL", "123.456") 

164 self._assertToken(lexer.token(), "TIME_LITERAL", "time") 

165 self.assertIsNone(lexer.token()) 

166 

167 def testIdentifier(self): 

168 """Test for numeric literals""" 

169 lexer = ParserLex.make_lexer() 

170 

171 lexer.input("ID id _012 a_b_C") 

172 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "ID") 

173 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id") 

174 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "_012") 

175 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a_b_C") 

176 self.assertIsNone(lexer.token()) 

177 

178 lexer.input("a.b a.b.c _._ _._._") 

179 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b") 

180 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c") 

181 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._") 

182 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "_._._") 

183 self.assertIsNone(lexer.token()) 

184 

185 lexer.input(".id") 

186 with self.assertRaises(ParserLexError): 

187 lexer.token() 

188 

189 lexer.input("id.") 

190 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "id") 

191 with self.assertRaises(ParserLexError): 

192 lexer.token() 

193 

194 lexer.input("id.id.id.id") 

195 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "id.id.id") 

196 with self.assertRaises(ParserLexError): 

197 lexer.token() 

198 

199 def testExpression(self): 

200 """Test for more or less complete expression""" 

201 lexer = ParserLex.make_lexer() 

202 

203 expr = ( 

204 "((instrument='HSC' AND detector != 9) OR instrument='CFHT') " 

205 "AND tract=8766 AND patch.cell_x > 5 AND " 

206 "patch.cell_y < 4 AND band='i' " 

207 "or visit IN (1..50:2)" 

208 ) 

209 tokens = ( 

210 ("LPAREN", "("), 

211 ("LPAREN", "("), 

212 ("SIMPLE_IDENTIFIER", "instrument"), 

213 ("EQ", "="), 

214 ("STRING_LITERAL", "HSC"), 

215 ("AND", "AND"), 

216 ("SIMPLE_IDENTIFIER", "detector"), 

217 ("NE", "!="), 

218 ("NUMERIC_LITERAL", "9"), 

219 ("RPAREN", ")"), 

220 ("OR", "OR"), 

221 ("SIMPLE_IDENTIFIER", "instrument"), 

222 ("EQ", "="), 

223 ("STRING_LITERAL", "CFHT"), 

224 ("RPAREN", ")"), 

225 ("AND", "AND"), 

226 ("SIMPLE_IDENTIFIER", "tract"), 

227 ("EQ", "="), 

228 ("NUMERIC_LITERAL", "8766"), 

229 ("AND", "AND"), 

230 ("QUALIFIED_IDENTIFIER", "patch.cell_x"), 

231 ("GT", ">"), 

232 ("NUMERIC_LITERAL", "5"), 

233 ("AND", "AND"), 

234 ("QUALIFIED_IDENTIFIER", "patch.cell_y"), 

235 ("LT", "<"), 

236 ("NUMERIC_LITERAL", "4"), 

237 ("AND", "AND"), 

238 ("SIMPLE_IDENTIFIER", "band"), 

239 ("EQ", "="), 

240 ("STRING_LITERAL", "i"), 

241 ("OR", "OR"), 

242 ("SIMPLE_IDENTIFIER", "visit"), 

243 ("IN", "IN"), 

244 ("LPAREN", "("), 

245 ("RANGE_LITERAL", (1, 50, 2)), 

246 ("RPAREN", ")"), 

247 ) 

248 lexer.input(expr) 

249 for type, value in tokens: 

250 self._assertToken(lexer.token(), type, value) 

251 self.assertIsNone(lexer.token()) 

252 

253 def testExceptions(self): 

254 """Test for exception contents""" 

255 

256 def _assertExc(exc, expr, remain, pos, lineno): 

257 """Check exception attribute values""" 

258 self.assertEqual(exc.expression, expr) 

259 self.assertEqual(exc.remain, remain) 

260 self.assertEqual(exc.pos, pos) 

261 self.assertEqual(exc.lineno, lineno) 

262 

263 lexer = ParserLex.make_lexer() 

264 expr = "a.b.c.d" 

265 lexer.input(expr) 

266 self._assertToken(lexer.token(), "QUALIFIED_IDENTIFIER", "a.b.c") 

267 with self.assertRaises(ParserLexError) as catcher: 

268 lexer.token() 

269 _assertExc(catcher.exception, expr, ".d", 5, 1) 

270 

271 lexer = ParserLex.make_lexer() 

272 expr = "a \n& b" 

273 lexer.input(expr) 

274 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a") 

275 with self.assertRaises(ParserLexError) as catcher: 

276 lexer.token() 

277 _assertExc(catcher.exception, expr, "& b", 3, 2) 

278 

279 lexer = ParserLex.make_lexer() 

280 expr = "a\n=\n1e5.e2" 

281 lexer.input(expr) 

282 self._assertToken(lexer.token(), "SIMPLE_IDENTIFIER", "a") 

283 self._assertToken(lexer.token(), "EQ", "=") 

284 self._assertToken(lexer.token(), "NUMERIC_LITERAL", "1e5") 

285 with self.assertRaises(ParserLexError) as catcher: 

286 lexer.token() 

287 _assertExc(catcher.exception, expr, ".e2", 7, 3) 

288 

289 # zero stride in range literal 

290 lexer = ParserLex.make_lexer() 

291 expr = "1..2:0" 

292 lexer.input(expr) 

293 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None)) 

294 with self.assertRaises(ParserLexError) as catcher: 

295 lexer.token() 

296 _assertExc(catcher.exception, expr, ":0", 4, 1) 

297 

298 # negative stride in range literal 

299 lexer = ParserLex.make_lexer() 

300 expr = "1..2:-10" 

301 lexer.input(expr) 

302 self._assertToken(lexer.token(), "RANGE_LITERAL", (1, 2, None)) 

303 with self.assertRaises(ParserLexError) as catcher: 

304 lexer.token() 

305 _assertExc(catcher.exception, expr, ":-10", 4, 1) 

306 

307 

308if __name__ == "__main__": 

309 unittest.main()