Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for exprParser subpackage module. 

23""" 

24 

25import unittest 

26 

27import astropy.time 

28 

29from lsst.daf.butler.registry.queries.exprParser import exprTree, TreeVisitor, ParserYacc, ParseError 

30from lsst.daf.butler.registry.queries.exprParser.parserYacc import _parseTimeString 

31 

32 

33class _Visitor(TreeVisitor): 

34 """Trivial implementation of TreeVisitor. 

35 """ 

36 def visitNumericLiteral(self, value, node): 

37 return f"N({value})" 

38 

39 def visitStringLiteral(self, value, node): 

40 return f"S({value})" 

41 

42 def visitTimeLiteral(self, value, node): 

43 return f"T({value})" 

44 

45 def visitRangeLiteral(self, start, stop, stride, node): 

46 if stride is None: 

47 return f"R({start}..{stop})" 

48 else: 

49 return f"R({start}..{stop}:{stride})" 

50 

51 def visitIdentifier(self, name, node): 

52 return f"ID({name})" 

53 

54 def visitUnaryOp(self, operator, operand, node): 

55 return f"U({operator} {operand})" 

56 

57 def visitBinaryOp(self, operator, lhs, rhs, node): 

58 return f"B({lhs} {operator} {rhs})" 

59 

60 def visitIsIn(self, lhs, values, not_in, node): 

61 values = ", ".join([str(val) for val in values]) 

62 if not_in: 

63 return f"!IN({lhs} ({values}))" 

64 else: 

65 return f"IN({lhs} ({values}))" 

66 

67 def visitParens(self, expression, node): 

68 return f"P({expression})" 

69 

70 

71class ParserLexTestCase(unittest.TestCase): 

72 """A test case for ParserYacc 

73 """ 

74 

75 def setUp(self): 

76 pass 

77 

78 def tearDown(self): 

79 pass 

80 

81 def testInstantiate(self): 

82 """Tests for making ParserLex instances 

83 """ 

84 parser = ParserYacc() # noqa: F841 

85 

86 def testEmpty(self): 

87 """Tests for empty expression 

88 """ 

89 parser = ParserYacc() 

90 

91 # empty expression is allowed, returns None 

92 tree = parser.parse("") 

93 self.assertIsNone(tree) 

94 

95 def testParseLiteral(self): 

96 """Tests for literals (strings/numbers) 

97 """ 

98 parser = ParserYacc() 

99 

100 tree = parser.parse('1') 

101 self.assertIsInstance(tree, exprTree.NumericLiteral) 

102 self.assertEqual(tree.value, '1') 

103 

104 tree = parser.parse('.5e-2') 

105 self.assertIsInstance(tree, exprTree.NumericLiteral) 

106 self.assertEqual(tree.value, '.5e-2') 

107 

108 tree = parser.parse("'string'") 

109 self.assertIsInstance(tree, exprTree.StringLiteral) 

110 self.assertEqual(tree.value, 'string') 

111 

112 tree = parser.parse("10..20") 

113 self.assertIsInstance(tree, exprTree.RangeLiteral) 

114 self.assertEqual(tree.start, 10) 

115 self.assertEqual(tree.stop, 20) 

116 self.assertEqual(tree.stride, None) 

117 

118 tree = parser.parse("-10 .. 10:5") 

119 self.assertIsInstance(tree, exprTree.RangeLiteral) 

120 self.assertEqual(tree.start, -10) 

121 self.assertEqual(tree.stop, 10) 

122 self.assertEqual(tree.stride, 5) 

123 

124 # more extensive tests of time parsing is below 

125 tree = parser.parse("T'51544.0'") 

126 self.assertIsInstance(tree, exprTree.TimeLiteral) 

127 self.assertEqual(tree.value, astropy.time.Time(51544.0, format="mjd", scale="tai")) 

128 

129 tree = parser.parse("T'2020-03-30T12:20:33'") 

130 self.assertIsInstance(tree, exprTree.TimeLiteral) 

131 self.assertEqual(tree.value, astropy.time.Time("2020-03-30T12:20:33", format="isot", scale="utc")) 

132 

133 def testParseIdentifiers(self): 

134 """Tests for identifiers 

135 """ 

136 parser = ParserYacc() 

137 

138 tree = parser.parse('a') 

139 self.assertIsInstance(tree, exprTree.Identifier) 

140 self.assertEqual(tree.name, 'a') 

141 

142 tree = parser.parse('a.b') 

143 self.assertIsInstance(tree, exprTree.Identifier) 

144 self.assertEqual(tree.name, 'a.b') 

145 

146 def testParseParens(self): 

147 """Tests for identifiers 

148 """ 

149 parser = ParserYacc() 

150 

151 tree = parser.parse('(a)') 

152 self.assertIsInstance(tree, exprTree.Parens) 

153 self.assertIsInstance(tree.expr, exprTree.Identifier) 

154 self.assertEqual(tree.expr.name, 'a') 

155 

156 def testUnaryOps(self): 

157 """Tests for unary plus and minus 

158 """ 

159 parser = ParserYacc() 

160 

161 tree = parser.parse('+a') 

162 self.assertIsInstance(tree, exprTree.UnaryOp) 

163 self.assertEqual(tree.op, '+') 

164 self.assertIsInstance(tree.operand, exprTree.Identifier) 

165 self.assertEqual(tree.operand.name, 'a') 

166 

167 tree = parser.parse('- x.y') 

168 self.assertIsInstance(tree, exprTree.UnaryOp) 

169 self.assertEqual(tree.op, '-') 

170 self.assertIsInstance(tree.operand, exprTree.Identifier) 

171 self.assertEqual(tree.operand.name, 'x.y') 

172 

173 def testBinaryOps(self): 

174 """Tests for binary operators 

175 """ 

176 parser = ParserYacc() 

177 

178 tree = parser.parse('a + b') 

179 self.assertIsInstance(tree, exprTree.BinaryOp) 

180 self.assertEqual(tree.op, '+') 

181 self.assertIsInstance(tree.lhs, exprTree.Identifier) 

182 self.assertIsInstance(tree.rhs, exprTree.Identifier) 

183 self.assertEqual(tree.lhs.name, 'a') 

184 self.assertEqual(tree.rhs.name, 'b') 

185 

186 tree = parser.parse('a - 2') 

187 self.assertIsInstance(tree, exprTree.BinaryOp) 

188 self.assertEqual(tree.op, '-') 

189 self.assertIsInstance(tree.lhs, exprTree.Identifier) 

190 self.assertIsInstance(tree.rhs, exprTree.NumericLiteral) 

191 self.assertEqual(tree.lhs.name, 'a') 

192 self.assertEqual(tree.rhs.value, '2') 

193 

194 tree = parser.parse('2 * 2') 

195 self.assertIsInstance(tree, exprTree.BinaryOp) 

196 self.assertEqual(tree.op, '*') 

197 self.assertIsInstance(tree.lhs, exprTree.NumericLiteral) 

198 self.assertIsInstance(tree.rhs, exprTree.NumericLiteral) 

199 self.assertEqual(tree.lhs.value, '2') 

200 self.assertEqual(tree.rhs.value, '2') 

201 

202 tree = parser.parse('1.e5/2') 

203 self.assertIsInstance(tree, exprTree.BinaryOp) 

204 self.assertEqual(tree.op, '/') 

205 self.assertIsInstance(tree.lhs, exprTree.NumericLiteral) 

206 self.assertIsInstance(tree.rhs, exprTree.NumericLiteral) 

207 self.assertEqual(tree.lhs.value, '1.e5') 

208 self.assertEqual(tree.rhs.value, '2') 

209 

210 tree = parser.parse('333%76') 

211 self.assertIsInstance(tree, exprTree.BinaryOp) 

212 self.assertEqual(tree.op, '%') 

213 self.assertIsInstance(tree.lhs, exprTree.NumericLiteral) 

214 self.assertIsInstance(tree.rhs, exprTree.NumericLiteral) 

215 self.assertEqual(tree.lhs.value, '333') 

216 self.assertEqual(tree.rhs.value, '76') 

217 

218 def testIsIn(self): 

219 """Tests for IN 

220 """ 

221 parser = ParserYacc() 

222 

223 tree = parser.parse("a in (1,2,'X')") 

224 self.assertIsInstance(tree, exprTree.IsIn) 

225 self.assertFalse(tree.not_in) 

226 self.assertIsInstance(tree.lhs, exprTree.Identifier) 

227 self.assertEqual(tree.lhs.name, 'a') 

228 self.assertIsInstance(tree.values, list) 

229 self.assertEqual(len(tree.values), 3) 

230 self.assertIsInstance(tree.values[0], exprTree.NumericLiteral) 

231 self.assertEqual(tree.values[0].value, '1') 

232 self.assertIsInstance(tree.values[1], exprTree.NumericLiteral) 

233 self.assertEqual(tree.values[1].value, '2') 

234 self.assertIsInstance(tree.values[2], exprTree.StringLiteral) 

235 self.assertEqual(tree.values[2].value, 'X') 

236 

237 tree = parser.parse("10 not in (1000, 2000..3000:100)") 

238 self.assertIsInstance(tree, exprTree.IsIn) 

239 self.assertTrue(tree.not_in) 

240 self.assertIsInstance(tree.lhs, exprTree.NumericLiteral) 

241 self.assertEqual(tree.lhs.value, '10') 

242 self.assertIsInstance(tree.values, list) 

243 self.assertEqual(len(tree.values), 2) 

244 self.assertIsInstance(tree.values[0], exprTree.NumericLiteral) 

245 self.assertEqual(tree.values[0].value, '1000') 

246 self.assertIsInstance(tree.values[1], exprTree.RangeLiteral) 

247 self.assertEqual(tree.values[1].start, 2000) 

248 self.assertEqual(tree.values[1].stop, 3000) 

249 self.assertEqual(tree.values[1].stride, 100) 

250 

251 tree = parser.parse("10 in (-1000, -2000)") 

252 self.assertIsInstance(tree, exprTree.IsIn) 

253 self.assertFalse(tree.not_in) 

254 self.assertIsInstance(tree.lhs, exprTree.NumericLiteral) 

255 self.assertEqual(tree.lhs.value, '10') 

256 self.assertIsInstance(tree.values, list) 

257 self.assertEqual(len(tree.values), 2) 

258 self.assertIsInstance(tree.values[0], exprTree.NumericLiteral) 

259 self.assertEqual(tree.values[0].value, '-1000') 

260 self.assertIsInstance(tree.values[1], exprTree.NumericLiteral) 

261 self.assertEqual(tree.values[1].value, '-2000') 

262 

263 def testCompareOps(self): 

264 """Tests for comparison operators 

265 """ 

266 parser = ParserYacc() 

267 

268 for op in ('=', '!=', '<', '<=', '>', '>='): 

269 tree = parser.parse('a {} 10'.format(op)) 

270 self.assertIsInstance(tree, exprTree.BinaryOp) 

271 self.assertEqual(tree.op, op) 

272 self.assertIsInstance(tree.lhs, exprTree.Identifier) 

273 self.assertIsInstance(tree.rhs, exprTree.NumericLiteral) 

274 self.assertEqual(tree.lhs.name, 'a') 

275 self.assertEqual(tree.rhs.value, '10') 

276 

277 def testBoolOps(self): 

278 """Tests for boolean operators 

279 """ 

280 parser = ParserYacc() 

281 

282 for op in ('OR', 'AND'): 

283 tree = parser.parse('a {} b'.format(op)) 

284 self.assertIsInstance(tree, exprTree.BinaryOp) 

285 self.assertEqual(tree.op, op) 

286 self.assertIsInstance(tree.lhs, exprTree.Identifier) 

287 self.assertIsInstance(tree.rhs, exprTree.Identifier) 

288 self.assertEqual(tree.lhs.name, 'a') 

289 self.assertEqual(tree.rhs.name, 'b') 

290 

291 tree = parser.parse('NOT b') 

292 self.assertIsInstance(tree, exprTree.UnaryOp) 

293 self.assertEqual(tree.op, 'NOT') 

294 self.assertIsInstance(tree.operand, exprTree.Identifier) 

295 self.assertEqual(tree.operand.name, 'b') 

296 

297 def testExpression(self): 

298 """Test for more or less complete expression""" 

299 parser = ParserYacc() 

300 

301 expression = ("((instrument='HSC' AND detector != 9) OR instrument='CFHT') " 

302 "AND tract=8766 AND patch.cell_x > 5 AND " 

303 "patch.cell_y < 4 AND band='i'") 

304 

305 tree = parser.parse(expression) 

306 self.assertIsInstance(tree, exprTree.BinaryOp) 

307 self.assertEqual(tree.op, 'AND') 

308 self.assertIsInstance(tree.lhs, exprTree.BinaryOp) 

309 # AND is left-associative, so rhs operand will be the 

310 # last sub-expressions 

311 self.assertIsInstance(tree.rhs, exprTree.BinaryOp) 

312 self.assertEqual(tree.rhs.op, '=') 

313 self.assertIsInstance(tree.rhs.lhs, exprTree.Identifier) 

314 self.assertEqual(tree.rhs.lhs.name, 'band') 

315 self.assertIsInstance(tree.rhs.rhs, exprTree.StringLiteral) 

316 self.assertEqual(tree.rhs.rhs.value, 'i') 

317 

318 def testException(self): 

319 """Test for exceptional cases""" 

320 

321 def _assertExc(exc, expr, token, pos, lineno, posInLine): 

322 """Check exception attribute values""" 

323 self.assertEqual(exc.expression, expr) 

324 self.assertEqual(exc.token, token) 

325 self.assertEqual(exc.pos, pos) 

326 self.assertEqual(exc.lineno, lineno) 

327 self.assertEqual(exc.posInLine, posInLine) 

328 

329 parser = ParserYacc() 

330 

331 expression = "(1, 2, 3)" 

332 with self.assertRaises(ParseError) as catcher: 

333 parser.parse(expression) 

334 _assertExc(catcher.exception, expression, ",", 2, 1, 2) 

335 

336 expression = "\n(1\n,\n 2, 3)" 

337 with self.assertRaises(ParseError) as catcher: 

338 parser.parse(expression) 

339 _assertExc(catcher.exception, expression, ",", 4, 3, 0) 

340 

341 expression = "T'not-a-time'" 

342 with self.assertRaises(ParseError) as catcher: 

343 parser.parse(expression) 

344 _assertExc(catcher.exception, expression, "not-a-time", 0, 1, 0) 

345 

346 def testStr(self): 

347 """Test for formatting""" 

348 parser = ParserYacc() 

349 

350 tree = parser.parse("(a+b)") 

351 self.assertEqual(str(tree), '(a + b)') 

352 

353 tree = parser.parse("1 in (1,'x',3)") 

354 self.assertEqual(str(tree), "1 IN (1, 'x', 3)") 

355 

356 tree = parser.parse("a not in (1,'x',3)") 

357 self.assertEqual(str(tree), "a NOT IN (1, 'x', 3)") 

358 

359 tree = parser.parse("(A or B) And NoT (x+3 > y)") 

360 self.assertEqual(str(tree), "(A OR B) AND NOT (x + 3 > y)") 

361 

362 tree = parser.parse("A in (100, 200..300:50)") 

363 self.assertEqual(str(tree), "A IN (100, 200..300:50)") 

364 

365 def testVisit(self): 

366 """Test for visitor methods""" 

367 

368 # test should cover all visit* methods 

369 parser = ParserYacc() 

370 visitor = _Visitor() 

371 

372 tree = parser.parse("(a+b)") 

373 result = tree.visit(visitor) 

374 self.assertEqual(result, "P(B(ID(a) + ID(b)))") 

375 

376 tree = parser.parse("(A or B) and not (x + 3 > y)") 

377 result = tree.visit(visitor) 

378 self.assertEqual(result, "B(P(B(ID(A) OR ID(B))) AND U(NOT P(B(B(ID(x) + N(3)) > ID(y)))))") 

379 

380 tree = parser.parse("x in (1,2) AND y NOT IN (1.1, .25, 1e2) OR z in ('a', 'b')") 

381 result = tree.visit(visitor) 

382 self.assertEqual(result, "B(B(IN(ID(x) (N(1), N(2))) AND !IN(ID(y) (N(1.1), N(.25), N(1e2))))" 

383 " OR IN(ID(z) (S(a), S(b))))") 

384 

385 tree = parser.parse("x in (1,2,5..15) AND y NOT IN (-100..100:10)") 

386 result = tree.visit(visitor) 

387 self.assertEqual(result, "B(IN(ID(x) (N(1), N(2), R(5..15))) AND !IN(ID(y) (R(-100..100:10))))") 

388 

389 tree = parser.parse("time > T'2020-03-30'") 

390 result = tree.visit(visitor) 

391 self.assertEqual(result, "B(ID(time) > T(2020-03-30 00:00:00.000))") 

392 

393 def testParseTimeStr(self): 

394 """Test for _parseTimeString method""" 

395 

396 # few expected failures 

397 bad_times = [ 

398 "", 

399 " ", 

400 "123.456e10", # no exponents 

401 "mjd-dj/123.456", # format can only have word chars 

402 "123.456/mai-tai", # scale can only have word chars 

403 "2020-03-01 00", # iso needs minutes if hour is given 

404 "2020-03-01T", # isot needs hour:minute 

405 "2020:100:12", # yday needs minutes if hour is given 

406 "format/123456.00", # unknown format 

407 "123456.00/unscale", # unknown scale 

408 ] 

409 for bad_time in bad_times: 

410 with self.assertRaises(ValueError): 

411 _parseTimeString(bad_time) 

412 

413 # each tuple is (string, value, format, scale) 

414 tests = [ 

415 ("51544.0", 51544.0, "mjd", "tai"), 

416 ("mjd/51544.0", 51544.0, "mjd", "tai"), 

417 ("51544.0/tai", 51544.0, "mjd", "tai"), 

418 ("mjd/51544.0/tai", 51544.0, "mjd", "tai"), 

419 ("MJd/51544.0/TAi", 51544.0, "mjd", "tai"), 

420 ("jd/2451544.5", 2451544.5, "jd", "tai"), 

421 ("jd/2451544.5", 2451544.5, "jd", "tai"), 

422 ("51544.0/utc", 51544.0, "mjd", "utc"), 

423 ("unix/946684800.0", 946684800., "unix", "utc"), 

424 ("cxcsec/63072064.184", 63072064.184, "cxcsec", "tt"), 

425 ("2020-03-30", "2020-03-30 00:00:00.000", "iso", "utc"), 

426 ("2020-03-30 12:20", "2020-03-30 12:20:00.000", "iso", "utc"), 

427 ("2020-03-30 12:20:33.456789", "2020-03-30 12:20:33.457", "iso", "utc"), 

428 ("2020-03-30T12:20", "2020-03-30T12:20:00.000", "isot", "utc"), 

429 ("2020-03-30T12:20:33.456789", "2020-03-30T12:20:33.457", "isot", "utc"), 

430 ("isot/2020-03-30", "2020-03-30T00:00:00.000", "isot", "utc"), 

431 ("2020-03-30/tai", "2020-03-30 00:00:00.000", "iso", "tai"), 

432 ("+02020-03-30", "2020-03-30T00:00:00.000", "fits", "utc"), 

433 ("+02020-03-30T12:20:33", "2020-03-30T12:20:33.000", "fits", "utc"), 

434 ("+02020-03-30T12:20:33.456789", "2020-03-30T12:20:33.457", "fits", "utc"), 

435 ("fits/2020-03-30", "2020-03-30T00:00:00.000", "fits", "utc"), 

436 ("2020:123", "2020:123:00:00:00.000", "yday", "utc"), 

437 ("2020:123:12:20", "2020:123:12:20:00.000", "yday", "utc"), 

438 ("2020:123:12:20:33.456789", "2020:123:12:20:33.457", "yday", "utc"), 

439 ("yday/2020:123:12:20/tai", "2020:123:12:20:00.000", "yday", "tai"), 

440 ] 

441 for time_str, value, fmt, scale in tests: 

442 time = _parseTimeString(time_str) 

443 self.assertEqual(time.value, value) 

444 self.assertEqual(time.format, fmt) 

445 self.assertEqual(time.scale, scale) 

446 

447 

448if __name__ == "__main__": 448 ↛ 449line 448 didn't jump to line 449, because the condition on line 448 was never true

449 unittest.main()