Coverage for tests/test_pipeTools.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

107 statements  

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for Pipeline. 

23""" 

24 

25import unittest 

26 

27from lsst.pipe.base import (PipelineTask, PipelineTaskConfig, 

28 PipelineTaskConnections, Pipeline, 

29 pipeTools) 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.utils.tests 

32 

33 

34class ExamplePipelineTaskConnections(PipelineTaskConnections, dimensions=["Visit", "Detector"]): 

35 input1 = cT.Input(name="", 

36 dimensions=["Visit", "Detector"], 

37 storageClass="example", 

38 doc="Input for this task") 

39 input2 = cT.Input(name="", 

40 dimensions=["Visit", "Detector"], 

41 storageClass="example", 

42 doc="Input for this task") 

43 output1 = cT.Output(name="", 

44 dimensions=["Visit", "Detector"], 

45 storageClass="example", 

46 doc="Output for this task") 

47 output2 = cT.Output(name="", 

48 dimensions=["Visit", "Detector"], 

49 storageClass="example", 

50 doc="Output for this task") 

51 

52 def __init__(self, *, config=None): 

53 super().__init__(config=config) 

54 if not config.connections.input2: 

55 self.inputs.remove('input2') 

56 if not config.connections.output2: 

57 self.outputs.remove('output2') 

58 

59 

60class ExamplePipelineTaskConfig(PipelineTaskConfig, pipelineConnections=ExamplePipelineTaskConnections): 

61 pass 

62 

63 

64def _makeConfig(inputName, outputName, pipeline, label): 

65 """Factory method for config instances 

66 

67 inputName and outputName can be either string or tuple of strings 

68 with two items max. 

69 """ 

70 if isinstance(inputName, tuple): 

71 pipeline.addConfigOverride(label, "connections.input1", inputName[0]) 

72 pipeline.addConfigOverride(label, "connections.input2", inputName[1] if len(inputName) > 1 else "") 

73 else: 

74 pipeline.addConfigOverride(label, "connections.input1", inputName) 

75 

76 if isinstance(outputName, tuple): 

77 pipeline.addConfigOverride(label, "connections.output1", outputName[0]) 

78 pipeline.addConfigOverride(label, "connections.output2", outputName[1] if len(outputName) > 1 else "") 

79 else: 

80 pipeline.addConfigOverride(label, "connections.output1", outputName) 

81 

82 

83class ExamplePipelineTask(PipelineTask): 

84 ConfigClass = ExamplePipelineTaskConfig 

85 

86 

87def _makePipeline(tasks): 

88 """Generate Pipeline instance. 

89 

90 Parameters 

91 ---------- 

92 tasks : list of tuples 

93 Each tuple in the list has 3 or 4 items: 

94 - input DatasetType name(s), string or tuple of strings 

95 - output DatasetType name(s), string or tuple of strings 

96 - task label, string 

97 - optional task class object, can be None 

98 

99 Returns 

100 ------- 

101 Pipeline instance 

102 """ 

103 pipe = Pipeline("test pipeline") 

104 for task in tasks: 

105 inputs = task[0] 

106 outputs = task[1] 

107 label = task[2] 

108 klass = task[3] if len(task) > 3 else ExamplePipelineTask 

109 pipe.addTask(klass, label) 

110 _makeConfig(inputs, outputs, pipe, label) 

111 return list(pipe.toExpandedPipeline()) 

112 

113 

114class PipelineToolsTestCase(unittest.TestCase): 

115 """A test case for pipelineTools 

116 """ 

117 

118 def setUp(self): 

119 pass 

120 

121 def tearDown(self): 

122 pass 

123 

124 def testIsOrdered(self): 

125 """Tests for pipeTools.isPipelineOrdered method 

126 """ 

127 pipeline = _makePipeline([("A", "B", "task1"), 

128 ("B", "C", "task2")]) 

129 self.assertTrue(pipeTools.isPipelineOrdered(pipeline)) 

130 

131 pipeline = _makePipeline([("A", ("B", "C"), "task1"), 

132 ("B", "D", "task2"), 

133 ("C", "E", "task3"), 

134 (("D", "E"), "F", "task4")]) 

135 self.assertTrue(pipeTools.isPipelineOrdered(pipeline)) 

136 

137 pipeline = _makePipeline([("A", ("B", "C"), "task1"), 

138 ("C", "E", "task2"), 

139 ("B", "D", "task3"), 

140 (("D", "E"), "F", "task4")]) 

141 self.assertTrue(pipeTools.isPipelineOrdered(pipeline)) 

142 

143 def testIsOrderedExceptions(self): 

144 """Tests for pipeTools.isPipelineOrdered method exceptions 

145 """ 

146 # two producers should throw ValueError 

147 with self.assertRaises(pipeTools.DuplicateOutputError): 

148 _makePipeline([("A", "B", "task1"), 

149 ("B", "C", "task2"), 

150 ("A", "C", "task3"), 

151 ]) 

152 

153 def testOrderPipeline(self): 

154 """Tests for pipeTools.orderPipeline method 

155 """ 

156 pipeline = _makePipeline([("A", "B", "task1"), 

157 ("B", "C", "task2")]) 

158 pipeline = pipeTools.orderPipeline(pipeline) 

159 self.assertEqual(len(pipeline), 2) 

160 self.assertEqual(pipeline[0].label, "task1") 

161 self.assertEqual(pipeline[1].label, "task2") 

162 

163 pipeline = _makePipeline([("B", "C", "task2"), 

164 ("A", "B", "task1")]) 

165 pipeline = pipeTools.orderPipeline(pipeline) 

166 self.assertEqual(len(pipeline), 2) 

167 self.assertEqual(pipeline[0].label, "task1") 

168 self.assertEqual(pipeline[1].label, "task2") 

169 

170 pipeline = _makePipeline([("A", ("B", "C"), "task1"), 

171 ("B", "D", "task2"), 

172 ("C", "E", "task3"), 

173 (("D", "E"), "F", "task4")]) 

174 pipeline = pipeTools.orderPipeline(pipeline) 

175 self.assertEqual(len(pipeline), 4) 

176 self.assertEqual(pipeline[0].label, "task1") 

177 self.assertEqual(pipeline[1].label, "task2") 

178 self.assertEqual(pipeline[2].label, "task3") 

179 self.assertEqual(pipeline[3].label, "task4") 

180 

181 pipeline = _makePipeline([("A", ("B", "C"), "task1"), 

182 ("C", "E", "task3"), 

183 ("B", "D", "task2"), 

184 (("D", "E"), "F", "task4")]) 

185 pipeline = pipeTools.orderPipeline(pipeline) 

186 self.assertEqual(len(pipeline), 4) 

187 self.assertEqual(pipeline[0].label, "task1") 

188 self.assertEqual(pipeline[1].label, "task3") 

189 self.assertEqual(pipeline[2].label, "task2") 

190 self.assertEqual(pipeline[3].label, "task4") 

191 

192 pipeline = _makePipeline([(("D", "E"), "F", "task4"), 

193 ("B", "D", "task2"), 

194 ("C", "E", "task3"), 

195 ("A", ("B", "C"), "task1")]) 

196 pipeline = pipeTools.orderPipeline(pipeline) 

197 self.assertEqual(len(pipeline), 4) 

198 self.assertEqual(pipeline[0].label, "task1") 

199 self.assertEqual(pipeline[1].label, "task2") 

200 self.assertEqual(pipeline[2].label, "task3") 

201 self.assertEqual(pipeline[3].label, "task4") 

202 

203 pipeline = _makePipeline([(("D", "E"), "F", "task4"), 

204 ("C", "E", "task3"), 

205 ("B", "D", "task2"), 

206 ("A", ("B", "C"), "task1")]) 

207 pipeline = pipeTools.orderPipeline(pipeline) 

208 self.assertEqual(len(pipeline), 4) 

209 self.assertEqual(pipeline[0].label, "task1") 

210 self.assertEqual(pipeline[1].label, "task3") 

211 self.assertEqual(pipeline[2].label, "task2") 

212 self.assertEqual(pipeline[3].label, "task4") 

213 

214 def testOrderPipelineExceptions(self): 

215 """Tests for pipeTools.orderPipeline method exceptions 

216 """ 

217 with self.assertRaises(pipeTools.DuplicateOutputError): 

218 _makePipeline([("A", "B", "task1"), 

219 ("B", "C", "task2"), 

220 ("A", "C", "task3"), 

221 ]) 

222 

223 # cycle in a graph should throw ValueError 

224 with self.assertRaises(pipeTools.PipelineDataCycleError): 

225 _makePipeline([("A", ("A", "B"), "task1")]) 

226 

227 # another kind of cycle in a graph 

228 with self.assertRaises(pipeTools.PipelineDataCycleError): 

229 _makePipeline([("A", "B", "task1"), 

230 ("B", "C", "task2"), 

231 ("C", "D", "task3"), 

232 ("D", "A", "task4")]) 

233 

234 

235class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

236 pass 

237 

238 

239def setup_module(module): 

240 lsst.utils.tests.init() 

241 

242 

243if __name__ == "__main__": 243 ↛ 244line 243 didn't jump to line 244, because the condition on line 243 was never true

244 lsst.utils.tests.init() 

245 unittest.main()