Coverage for tests/test_projection.py: 19%

55 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-19 09:55 +0000

1# This file is part of daf_relation. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import unittest 

25 

26from lsst.daf.relation import ( 

27 ColumnError, 

28 ColumnExpression, 

29 EngineError, 

30 Projection, 

31 SortTerm, 

32 UnaryOperationRelation, 

33 iteration, 

34 tests, 

35) 

36 

37 

38class ProjectionTestCase(tests.RelationTestCase): 

39 """Tests for the Projection operation and relations based on it.""" 

40 

41 def setUp(self) -> None: 

42 self.a = tests.ColumnTag("a") 

43 self.b = tests.ColumnTag("b") 

44 self.c = tests.ColumnTag("c") 

45 self.engine = iteration.Engine(name="preferred") 

46 self.leaf = self.engine.make_leaf( 

47 {self.a, self.b, self.c}, 

48 payload=iteration.RowSequence( 

49 [{self.a: 1, self.b: 4}, {self.a: 0, self.b: 5}, {self.a: 1, self.b: 6}] 

50 ), 

51 name="leaf", 

52 ) 

53 

54 def test_attributes(self) -> None: 

55 """Check that all UnaryOperation and Relation attributes have the 

56 expected values. 

57 """ 

58 relation = self.leaf.with_only_columns({self.a}) 

59 assert isinstance(relation, UnaryOperationRelation) 

60 self.assertEqual(relation.columns, {self.a}) 

61 self.assertEqual(relation.engine, self.engine) 

62 self.assertEqual(relation.min_rows, self.leaf.min_rows) 

63 self.assertEqual(relation.max_rows, self.leaf.max_rows) 

64 operation = relation.operation 

65 assert isinstance(operation, Projection) 

66 self.assertEqual(operation.columns, {self.a}) 

67 self.assertEqual(operation.columns_required, {self.a}) 

68 self.assertTrue(operation.is_empty_invariant) 

69 self.assertTrue(operation.is_count_invariant) 

70 self.assertFalse(operation.is_order_dependent) 

71 self.assertFalse(operation.is_count_dependent) 

72 

73 def test_apply_failures(self) -> None: 

74 """Test failure modes of constructing and applying Projections.""" 

75 # Required columns must be present. 

76 with self.assertRaises(ColumnError): 

77 self.leaf.with_only_columns({self.a, tests.ColumnTag("d")}) 

78 

79 def test_backtracking_apply(self) -> None: 

80 """Test apply logic that involves reordering operations in the existing 

81 tree to perform the new operation in a preferred engine. 

82 """ 

83 new_engine = iteration.Engine(name="downstream") 

84 d = tests.ColumnTag("d") 

85 predicate = ColumnExpression.reference(self.a).lt(ColumnExpression.literal(2)) 

86 expression = ColumnExpression.function( 

87 "__add__", ColumnExpression.reference(self.a), ColumnExpression.literal(5) 

88 ) 

89 sort_terms = [SortTerm(ColumnExpression.reference(self.a))] 

90 # Apply a bunch of operations in a new engine that a Projection should 

91 # commute with. 

92 target = ( 

93 self.leaf.transferred_to(new_engine) 

94 .with_calculated_column(d, expression) 

95 .with_rows_satisfying(predicate) 

96 .without_duplicates() 

97 .sorted(sort_terms) 

98 )[:2] 

99 # Apply a Projection to just {self.a} and check that it: 

100 # - appears before the transfer 

101 # - results in the Calculation being dropped (since that now does 

102 # nothing). 

103 self.assert_relations_equal( 

104 target.with_only_columns({self.a}, preferred_engine=self.engine, require_preferred_engine=True), 

105 ( 

106 self.leaf.with_only_columns({self.a}) 

107 .transferred_to(new_engine) 

108 .with_rows_satisfying(predicate) 

109 .without_duplicates() 

110 .sorted(sort_terms) 

111 )[:2], 

112 ) 

113 # Apply a Projection to {self.b}, which cannot be moved past the 

114 # calculation or selection in its entirety, since they all depend on 

115 # {self.a}. We should move as much of the projection as possible as 

116 # far upstream as possible, and leave the rest to be applied in the 

117 # non-preferred engine. 

118 self.assert_relations_equal( 

119 target.with_only_columns({self.b}, preferred_engine=self.engine), 

120 ( 

121 self.leaf.with_only_columns({self.a, self.b}) 

122 .transferred_to(new_engine) 

123 .with_rows_satisfying(predicate) 

124 .without_duplicates() 

125 .sorted(sort_terms) 

126 )[:2].with_only_columns({self.b}), 

127 ) 

128 # Similar attempt with Projection to {self.c}, this time with a 

129 # transfer request. 

130 self.assert_relations_equal( 

131 target.with_only_columns({self.c}, preferred_engine=self.engine, transfer=True), 

132 ( 

133 self.leaf.with_only_columns({self.a, self.c}) 

134 .transferred_to(new_engine) 

135 .with_rows_satisfying(predicate) 

136 .without_duplicates() 

137 .sorted(sort_terms) 

138 )[:2] 

139 .transferred_to(self.engine) 

140 .with_only_columns({self.c}), 

141 ) 

142 # Test that a projection that does nothing is simplified away. 

143 self.assert_relations_equal(self.leaf.with_only_columns(self.leaf.columns), self.leaf) 

144 # Test that back-to-back projections and do-nothing calculations are 

145 # simplified away, regardless of whether they are before or after the 

146 # preferred-engine transfer. 

147 target = self.leaf.with_calculated_column(d, expression).transferred_to(new_engine) 

148 self.assert_relations_equal( 

149 target.with_only_columns({self.a, d}, preferred_engine=self.engine).with_only_columns( 

150 {self.a}, preferred_engine=self.engine 

151 ), 

152 target.with_only_columns({self.a}, preferred_engine=self.engine), 

153 ) 

154 self.assert_relations_equal( 

155 target.with_only_columns({self.a, d}).with_only_columns({self.a}, preferred_engine=self.engine), 

156 target.with_only_columns({self.a}, preferred_engine=self.engine), 

157 ) 

158 

159 def test_no_backtracking(self) -> None: 

160 """Test apply logic that handles preferred engines without reordering 

161 operations in the existing tree. 

162 """ 

163 new_engine = iteration.Engine(name="downstream") 

164 # Construct a relation tree we can't reorder when inserting a 

165 # Projection, because there is a locked Materialization in the way. 

166 target = self.leaf.transferred_to(new_engine).materialized("lock") 

167 # Preferred engine is ignored if we can't backtrack and don't enable 

168 # anything else. 

169 self.assert_relations_equal( 

170 target.with_only_columns({self.a}, preferred_engine=self.engine), 

171 target.with_only_columns({self.a}), 

172 ) 

173 # We can force this to be an error. 

174 with self.assertRaises(EngineError): 

175 target.with_only_columns({self.a}, preferred_engine=self.engine, require_preferred_engine=True) 

176 # We can also automatically transfer (back) to the preferred engine. 

177 self.assert_relations_equal( 

178 target.with_only_columns({self.a}, preferred_engine=self.engine, transfer=True), 

179 target.transferred_to(self.engine).with_only_columns({self.a}), 

180 ) 

181 

182 def test_iteration(self) -> None: 

183 """Test Projection execution in the iteration engine.""" 

184 relation = self.leaf.with_only_columns({self.a}) 

185 self.assertEqual( 

186 list(self.engine.execute(relation)), 

187 [{self.a: 1}, {self.a: 0}, {self.a: 1}], 

188 ) 

189 

190 def test_str(self) -> None: 

191 """Test str(Projection) and 

192 str(UnaryOperationRelation[Projection]). 

193 """ 

194 relation = self.leaf.with_only_columns({self.a}) 

195 self.assertEqual(str(relation), "Π[a](leaf)") 

196 

197 

198if __name__ == "__main__": 

199 unittest.main()