Coverage for tests/test_pipeTools.py: 23%

108 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-19 04:46 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for Pipeline. 

23""" 

24 

25import unittest 

26 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.utils.tests 

29from lsst.pipe.base import Pipeline, PipelineTask, PipelineTaskConfig, PipelineTaskConnections, pipeTools 

30 

31 

32class ExamplePipelineTaskConnections(PipelineTaskConnections, dimensions=["Visit", "Detector"]): 

33 input1 = cT.Input( 

34 name="", dimensions=["Visit", "Detector"], storageClass="example", doc="Input for this task" 

35 ) 

36 input2 = cT.Input( 

37 name="", dimensions=["Visit", "Detector"], storageClass="example", doc="Input for this task" 

38 ) 

39 output1 = cT.Output( 

40 name="", dimensions=["Visit", "Detector"], storageClass="example", doc="Output for this task" 

41 ) 

42 output2 = cT.Output( 

43 name="", dimensions=["Visit", "Detector"], storageClass="example", doc="Output for this task" 

44 ) 

45 

46 def __init__(self, *, config=None): 

47 super().__init__(config=config) 

48 if not config.connections.input2: 

49 self.inputs.remove("input2") 

50 if not config.connections.output2: 

51 self.outputs.remove("output2") 

52 

53 

54class ExamplePipelineTaskConfig(PipelineTaskConfig, pipelineConnections=ExamplePipelineTaskConnections): 

55 pass 

56 

57 

58def _makeConfig(inputName, outputName, pipeline, label): 

59 """Factory method for config instances 

60 

61 inputName and outputName can be either string or tuple of strings 

62 with two items max. 

63 """ 

64 if isinstance(inputName, tuple): 

65 pipeline.addConfigOverride(label, "connections.input1", inputName[0]) 

66 pipeline.addConfigOverride(label, "connections.input2", inputName[1] if len(inputName) > 1 else "") 

67 else: 

68 pipeline.addConfigOverride(label, "connections.input1", inputName) 

69 

70 if isinstance(outputName, tuple): 

71 pipeline.addConfigOverride(label, "connections.output1", outputName[0]) 

72 pipeline.addConfigOverride(label, "connections.output2", outputName[1] if len(outputName) > 1 else "") 

73 else: 

74 pipeline.addConfigOverride(label, "connections.output1", outputName) 

75 

76 

77class ExamplePipelineTask(PipelineTask): 

78 ConfigClass = ExamplePipelineTaskConfig 

79 _DefaultName = "examplePipelineTask" 

80 

81 

82def _makePipeline(tasks): 

83 """Generate Pipeline instance. 

84 

85 Parameters 

86 ---------- 

87 tasks : list of tuples 

88 Each tuple in the list has 3 or 4 items: 

89 - input DatasetType name(s), string or tuple of strings 

90 - output DatasetType name(s), string or tuple of strings 

91 - task label, string 

92 - optional task class object, can be None 

93 

94 Returns 

95 ------- 

96 Pipeline instance 

97 """ 

98 pipe = Pipeline("test pipeline") 

99 for task in tasks: 

100 inputs = task[0] 

101 outputs = task[1] 

102 label = task[2] 

103 klass = task[3] if len(task) > 3 else ExamplePipelineTask 

104 pipe.addTask(klass, label) 

105 _makeConfig(inputs, outputs, pipe, label) 

106 return list(pipe.toExpandedPipeline()) 

107 

108 

109class PipelineToolsTestCase(unittest.TestCase): 

110 """A test case for pipelineTools""" 

111 

112 def setUp(self): 

113 pass 

114 

115 def tearDown(self): 

116 pass 

117 

118 def testIsOrdered(self): 

119 """Tests for pipeTools.isPipelineOrdered method""" 

120 pipeline = _makePipeline([("A", "B", "task1"), ("B", "C", "task2")]) 

121 self.assertTrue(pipeTools.isPipelineOrdered(pipeline)) 

122 

123 pipeline = _makePipeline( 

124 [("A", ("B", "C"), "task1"), ("B", "D", "task2"), ("C", "E", "task3"), (("D", "E"), "F", "task4")] 

125 ) 

126 self.assertTrue(pipeTools.isPipelineOrdered(pipeline)) 

127 

128 pipeline = _makePipeline( 

129 [("A", ("B", "C"), "task1"), ("C", "E", "task2"), ("B", "D", "task3"), (("D", "E"), "F", "task4")] 

130 ) 

131 self.assertTrue(pipeTools.isPipelineOrdered(pipeline)) 

132 

133 def testIsOrderedExceptions(self): 

134 """Tests for pipeTools.isPipelineOrdered method exceptions""" 

135 # two producers should throw ValueError 

136 with self.assertRaises(pipeTools.DuplicateOutputError): 

137 _makePipeline( 

138 [ 

139 ("A", "B", "task1"), 

140 ("B", "C", "task2"), 

141 ("A", "C", "task3"), 

142 ] 

143 ) 

144 

145 def testOrderPipeline(self): 

146 """Tests for pipeTools.orderPipeline method""" 

147 pipeline = _makePipeline([("A", "B", "task1"), ("B", "C", "task2")]) 

148 pipeline = pipeTools.orderPipeline(pipeline) 

149 self.assertEqual(len(pipeline), 2) 

150 self.assertEqual(pipeline[0].label, "task1") 

151 self.assertEqual(pipeline[1].label, "task2") 

152 

153 pipeline = _makePipeline([("B", "C", "task2"), ("A", "B", "task1")]) 

154 pipeline = pipeTools.orderPipeline(pipeline) 

155 self.assertEqual(len(pipeline), 2) 

156 self.assertEqual(pipeline[0].label, "task1") 

157 self.assertEqual(pipeline[1].label, "task2") 

158 

159 pipeline = _makePipeline( 

160 [("A", ("B", "C"), "task1"), ("B", "D", "task2"), ("C", "E", "task3"), (("D", "E"), "F", "task4")] 

161 ) 

162 pipeline = pipeTools.orderPipeline(pipeline) 

163 self.assertEqual(len(pipeline), 4) 

164 self.assertEqual(pipeline[0].label, "task1") 

165 self.assertEqual(pipeline[1].label, "task2") 

166 self.assertEqual(pipeline[2].label, "task3") 

167 self.assertEqual(pipeline[3].label, "task4") 

168 

169 pipeline = _makePipeline( 

170 [("A", ("B", "C"), "task1"), ("C", "E", "task3"), ("B", "D", "task2"), (("D", "E"), "F", "task4")] 

171 ) 

172 pipeline = pipeTools.orderPipeline(pipeline) 

173 self.assertEqual(len(pipeline), 4) 

174 self.assertEqual(pipeline[0].label, "task1") 

175 self.assertEqual(pipeline[1].label, "task2") 

176 self.assertEqual(pipeline[2].label, "task3") 

177 self.assertEqual(pipeline[3].label, "task4") 

178 

179 pipeline = _makePipeline( 

180 [(("D", "E"), "F", "task4"), ("B", "D", "task2"), ("C", "E", "task3"), ("A", ("B", "C"), "task1")] 

181 ) 

182 pipeline = pipeTools.orderPipeline(pipeline) 

183 self.assertEqual(len(pipeline), 4) 

184 self.assertEqual(pipeline[0].label, "task1") 

185 self.assertEqual(pipeline[1].label, "task2") 

186 self.assertEqual(pipeline[2].label, "task3") 

187 self.assertEqual(pipeline[3].label, "task4") 

188 

189 pipeline = _makePipeline( 

190 [(("D", "E"), "F", "task4"), ("C", "E", "task3"), ("B", "D", "task2"), ("A", ("B", "C"), "task1")] 

191 ) 

192 pipeline = pipeTools.orderPipeline(pipeline) 

193 self.assertEqual(len(pipeline), 4) 

194 self.assertEqual(pipeline[0].label, "task1") 

195 self.assertEqual(pipeline[1].label, "task2") 

196 self.assertEqual(pipeline[2].label, "task3") 

197 self.assertEqual(pipeline[3].label, "task4") 

198 

199 def testOrderPipelineExceptions(self): 

200 """Tests for pipeTools.orderPipeline method exceptions""" 

201 with self.assertRaises(pipeTools.DuplicateOutputError): 

202 _makePipeline( 

203 [ 

204 ("A", "B", "task1"), 

205 ("B", "C", "task2"), 

206 ("A", "C", "task3"), 

207 ] 

208 ) 

209 

210 # cycle in a graph should throw ValueError 

211 with self.assertRaises(pipeTools.PipelineDataCycleError): 

212 _makePipeline([("A", ("A", "B"), "task1")]) 

213 

214 # another kind of cycle in a graph 

215 with self.assertRaises(pipeTools.PipelineDataCycleError): 

216 _makePipeline( 

217 [("A", "B", "task1"), ("B", "C", "task2"), ("C", "D", "task3"), ("D", "A", "task4")] 

218 ) 

219 

220 

221class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

222 pass 

223 

224 

225def setup_module(module): 

226 lsst.utils.tests.init() 

227 

228 

229if __name__ == "__main__": 229 ↛ 230line 229 didn't jump to line 230, because the condition on line 229 was never true

230 lsst.utils.tests.init() 

231 unittest.main()