Coverage for tests/test_expressions.py: 19%

145 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-01 10:04 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import datetime 

23import unittest 

24 

25import astropy.time 

26import sqlalchemy 

27from lsst.daf.butler import ( 

28 ColumnTypeInfo, 

29 DataCoordinate, 

30 DatasetColumnTag, 

31 DimensionUniverse, 

32 TimespanDatabaseRepresentation, 

33 ddl, 

34 time_utils, 

35) 

36from lsst.daf.butler.registry.queries.expressions import make_string_expression_predicate 

37from lsst.daf.butler.registry.queries.expressions.check import CheckVisitor, InspectionVisitor 

38from lsst.daf.butler.registry.queries.expressions.normalForm import NormalForm, NormalFormExpression 

39from lsst.daf.butler.registry.queries.expressions.parser import ParserYacc 

40from lsst.daf.relation import ColumnContainer, ColumnExpression 

41from sqlalchemy.schema import Column 

42 

43 

44class FakeDatasetRecordStorageManager: 

45 ingestDate = Column("ingest_date") 

46 

47 

48class ConvertExpressionToPredicateTestCase(unittest.TestCase): 

49 """A test case for the make_string_expression_predicate function""" 

50 

51 def setUp(self): 

52 self.column_types = ColumnTypeInfo( 

53 timespan_cls=TimespanDatabaseRepresentation.Compound, 

54 universe=DimensionUniverse(), 

55 dataset_id_spec=ddl.FieldSpec("dataset_id", dtype=ddl.GUID), 

56 run_key_spec=ddl.FieldSpec("run_id", dtype=sqlalchemy.BigInteger), 

57 ) 

58 

59 def test_simple(self): 

60 """Test with a trivial expression""" 

61 self.assertEqual( 

62 make_string_expression_predicate("1 > 0", self.column_types.universe.empty)[0], 

63 ColumnExpression.literal(1, dtype=int).gt(ColumnExpression.literal(0, dtype=int)), 

64 ) 

65 

66 def test_time(self): 

67 """Test with a trivial expression including times""" 

68 time_converter = time_utils.TimeConverter() 

69 self.assertEqual( 

70 make_string_expression_predicate( 

71 "T'1970-01-01 00:00/tai' < T'2020-01-01 00:00/tai'", self.column_types.universe.empty 

72 )[0], 

73 ColumnExpression.literal(time_converter.nsec_to_astropy(0), dtype=astropy.time.Time).lt( 

74 ColumnExpression.literal( 

75 time_converter.nsec_to_astropy(1577836800000000000), dtype=astropy.time.Time 

76 ) 

77 ), 

78 ) 

79 

80 def test_ingest_date(self): 

81 """Test with an expression including ingest_date which is native UTC""" 

82 self.assertEqual( 

83 make_string_expression_predicate( 

84 "ingest_date < T'2020-01-01 00:00/utc'", 

85 self.column_types.universe.empty, 

86 dataset_type_name="fake", 

87 )[0], 

88 ColumnExpression.reference(DatasetColumnTag("fake", "ingest_date"), dtype=datetime.datetime).lt( 

89 ColumnExpression.literal(datetime.datetime(2020, 1, 1), dtype=datetime.datetime) 

90 ), 

91 ) 

92 

93 def test_bind(self): 

94 """Test with bind parameters""" 

95 

96 self.assertEqual( 

97 make_string_expression_predicate( 

98 "a > b OR t in (x, y, z)", 

99 self.column_types.universe.empty, 

100 bind={"a": 1, "b": 2, "t": 0, "x": 10, "y": 20, "z": 30}, 

101 )[0], 

102 ColumnExpression.literal(1, dtype=int) 

103 .gt(ColumnExpression.literal(2, dtype=int)) 

104 .logical_or( 

105 ColumnContainer.sequence( 

106 [ 

107 ColumnExpression.literal(10, dtype=int), 

108 ColumnExpression.literal(20, dtype=int), 

109 ColumnExpression.literal(30, dtype=int), 

110 ], 

111 dtype=int, 

112 ).contains(ColumnExpression.literal(0, dtype=int)) 

113 ), 

114 ) 

115 

116 def test_bind_list(self): 

117 """Test with bind parameter which is list/tuple/set inside IN rhs.""" 

118 

119 self.assertEqual( 

120 make_string_expression_predicate( 

121 "a > b OR t in (x)", 

122 self.column_types.universe.empty, 

123 bind={"a": 1, "b": 2, "t": 0, "x": (10, 20, 30)}, 

124 )[0], 

125 ColumnExpression.literal(1, dtype=int) 

126 .gt(ColumnExpression.literal(2, dtype=int)) 

127 .logical_or( 

128 ColumnContainer.sequence( 

129 [ 

130 ColumnExpression.literal(10, dtype=int), 

131 ColumnExpression.literal(20, dtype=int), 

132 ColumnExpression.literal(30, dtype=int), 

133 ], 

134 dtype=int, 

135 ).contains( 

136 ColumnExpression.literal(0, dtype=int), 

137 ) 

138 ), 

139 ) 

140 # Couple of bound variables inside IN() with different combinations 

141 # of scalars and list. 

142 self.assertEqual( 

143 make_string_expression_predicate( 

144 "a > b OR t in (x, y)", 

145 self.column_types.universe.empty, 

146 bind={"a": 1, "b": 2, "t": 0, "x": 10, "y": 20}, 

147 )[0], 

148 ColumnExpression.literal(1, dtype=int) 

149 .gt(ColumnExpression.literal(2, dtype=int)) 

150 .logical_or( 

151 ColumnContainer.sequence( 

152 [ 

153 ColumnExpression.literal(10, dtype=int), 

154 ColumnExpression.literal(20, dtype=int), 

155 ], 

156 dtype=int, 

157 ).contains( 

158 ColumnExpression.literal(0, dtype=int), 

159 ) 

160 ), 

161 ) 

162 self.assertEqual( 

163 make_string_expression_predicate( 

164 "a > b OR t in (x, y)", 

165 self.column_types.universe.empty, 

166 bind={"a": 1, "b": 2, "t": 0, "x": [10, 30], "y": 20}, 

167 )[0], 

168 ColumnExpression.literal(1, dtype=int) 

169 .gt(ColumnExpression.literal(2, dtype=int)) 

170 .logical_or( 

171 ColumnContainer.sequence( 

172 [ 

173 ColumnExpression.literal(10, dtype=int), 

174 ColumnExpression.literal(30, dtype=int), 

175 ColumnExpression.literal(20, dtype=int), 

176 ], 

177 dtype=int, 

178 ).contains( 

179 ColumnExpression.literal(0, dtype=int), 

180 ) 

181 ), 

182 ) 

183 self.assertEqual( 

184 make_string_expression_predicate( 

185 "a > b OR t in (x, y)", 

186 self.column_types.universe.empty, 

187 bind={"a": 1, "b": 2, "t": 0, "x": (10, 30), "y": {20}}, 

188 )[0], 

189 ColumnExpression.literal(1, dtype=int) 

190 .gt(ColumnExpression.literal(2, dtype=int)) 

191 .logical_or( 

192 ColumnContainer.sequence( 

193 [ 

194 ColumnExpression.literal(10, dtype=int), 

195 ColumnExpression.literal(30, dtype=int), 

196 ColumnExpression.literal(20, dtype=int), 

197 ], 

198 dtype=int, 

199 ).contains(ColumnExpression.literal(0, dtype=int)) 

200 ), 

201 ) 

202 

203 

204class InspectionVisitorTestCase(unittest.TestCase): 

205 """Tests for InspectionVisitor class.""" 

206 

207 def test_simple(self): 

208 """Test for simple expressions""" 

209 

210 universe = DimensionUniverse() 

211 parser = ParserYacc() 

212 

213 tree = parser.parse("instrument = 'LSST'") 

214 bind = {} 

215 summary = tree.visit(InspectionVisitor(universe, bind)) 

216 self.assertEqual(summary.dimensions.names, {"instrument"}) 

217 self.assertFalse(summary.columns) 

218 self.assertFalse(summary.hasIngestDate) 

219 self.assertEqual(summary.dataIdKey, universe["instrument"]) 

220 self.assertEqual(summary.dataIdValue, "LSST") 

221 

222 tree = parser.parse("instrument != 'LSST'") 

223 summary = tree.visit(InspectionVisitor(universe, bind)) 

224 self.assertEqual(summary.dimensions.names, {"instrument"}) 

225 self.assertFalse(summary.columns) 

226 self.assertIsNone(summary.dataIdKey) 

227 self.assertIsNone(summary.dataIdValue) 

228 

229 tree = parser.parse("instrument = 'LSST' AND visit = 1") 

230 summary = tree.visit(InspectionVisitor(universe, bind)) 

231 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

232 self.assertFalse(summary.columns) 

233 self.assertIsNone(summary.dataIdKey) 

234 self.assertIsNone(summary.dataIdValue) 

235 

236 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = 'x'") 

237 summary = tree.visit(InspectionVisitor(universe, bind)) 

238 self.assertEqual( 

239 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"} 

240 ) 

241 self.assertFalse(summary.columns) 

242 self.assertIsNone(summary.dataIdKey) 

243 self.assertIsNone(summary.dataIdValue) 

244 

245 def test_bind(self): 

246 """Test for simple expressions with binds.""" 

247 

248 universe = DimensionUniverse() 

249 parser = ParserYacc() 

250 

251 tree = parser.parse("instrument = instr") 

252 bind = {"instr": "LSST"} 

253 summary = tree.visit(InspectionVisitor(universe, bind)) 

254 self.assertEqual(summary.dimensions.names, {"instrument"}) 

255 self.assertFalse(summary.hasIngestDate) 

256 self.assertEqual(summary.dataIdKey, universe["instrument"]) 

257 self.assertEqual(summary.dataIdValue, "LSST") 

258 

259 tree = parser.parse("instrument != instr") 

260 self.assertEqual(summary.dimensions.names, {"instrument"}) 

261 summary = tree.visit(InspectionVisitor(universe, bind)) 

262 self.assertIsNone(summary.dataIdKey) 

263 self.assertIsNone(summary.dataIdValue) 

264 

265 tree = parser.parse("instrument = instr AND visit = visit_id") 

266 bind = {"instr": "LSST", "visit_id": 1} 

267 summary = tree.visit(InspectionVisitor(universe, bind)) 

268 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

269 self.assertIsNone(summary.dataIdKey) 

270 self.assertIsNone(summary.dataIdValue) 

271 

272 tree = parser.parse("instrument = 'LSST' AND visit = 1 AND skymap = skymap_name") 

273 bind = {"instr": "LSST", "visit_id": 1, "skymap_name": "x"} 

274 summary = tree.visit(InspectionVisitor(universe, bind)) 

275 self.assertEqual( 

276 summary.dimensions.names, {"instrument", "visit", "band", "physical_filter", "skymap"} 

277 ) 

278 self.assertIsNone(summary.dataIdKey) 

279 self.assertIsNone(summary.dataIdValue) 

280 

281 def test_in(self): 

282 """Test for IN expressions.""" 

283 

284 universe = DimensionUniverse() 

285 parser = ParserYacc() 

286 

287 tree = parser.parse("instrument IN ('LSST')") 

288 bind = {} 

289 summary = tree.visit(InspectionVisitor(universe, bind)) 

290 self.assertEqual(summary.dimensions.names, {"instrument"}) 

291 self.assertFalse(summary.hasIngestDate) 

292 # we do not handle IN with a single item as `=` 

293 self.assertIsNone(summary.dataIdKey) 

294 self.assertIsNone(summary.dataIdValue) 

295 

296 tree = parser.parse("instrument IN (instr)") 

297 bind = {"instr": "LSST"} 

298 summary = tree.visit(InspectionVisitor(universe, bind)) 

299 self.assertEqual(summary.dimensions.names, {"instrument"}) 

300 self.assertIsNone(summary.dataIdKey) 

301 self.assertIsNone(summary.dataIdValue) 

302 

303 tree = parser.parse("visit IN (1,2,3)") 

304 bind = {} 

305 summary = tree.visit(InspectionVisitor(universe, bind)) 

306 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

307 self.assertIsNone(summary.dataIdKey) 

308 self.assertIsNone(summary.dataIdValue) 

309 

310 tree = parser.parse("visit IN (visit1, visit2, visit3)") 

311 bind = {"visit1": 1, "visit2": 2, "visit3": 3} 

312 summary = tree.visit(InspectionVisitor(universe, bind)) 

313 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

314 self.assertIsNone(summary.dataIdKey) 

315 self.assertIsNone(summary.dataIdValue) 

316 

317 tree = parser.parse("visit IN (visits)") 

318 bind = {"visits": (1, 2, 3)} 

319 summary = tree.visit(InspectionVisitor(universe, bind)) 

320 self.assertEqual(summary.dimensions.names, {"instrument", "visit", "band", "physical_filter"}) 

321 self.assertIsNone(summary.dataIdKey) 

322 self.assertIsNone(summary.dataIdValue) 

323 

324 

325class CheckVisitorTestCase(unittest.TestCase): 

326 """Tests for CheckVisitor class.""" 

327 

328 def test_governor(self): 

329 """Test with governor dimension in expression""" 

330 

331 parser = ParserYacc() 

332 

333 universe = DimensionUniverse() 

334 graph = universe.extract(("instrument", "visit")) 

335 dataId = DataCoordinate.makeEmpty(universe) 

336 defaults = DataCoordinate.makeEmpty(universe) 

337 

338 # governor-only constraint 

339 tree = parser.parse("instrument = 'LSST'") 

340 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE) 

341 binds = {} 

342 visitor = CheckVisitor(dataId, graph, binds, defaults) 

343 expr.visit(visitor) 

344 

345 tree = parser.parse("'LSST' = instrument") 

346 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE) 

347 binds = {} 

348 visitor = CheckVisitor(dataId, graph, binds, defaults) 

349 expr.visit(visitor) 

350 

351 # use bind for governor 

352 tree = parser.parse("instrument = instr") 

353 expr = NormalFormExpression.fromTree(tree, NormalForm.DISJUNCTIVE) 

354 binds = {"instr": "LSST"} 

355 visitor = CheckVisitor(dataId, graph, binds, defaults) 

356 expr.visit(visitor) 

357 

358 

359if __name__ == "__main__": 359 ↛ 360line 359 didn't jump to line 360, because the condition on line 359 was never true

360 unittest.main()