Coverage for tests/test_dotTools.py: 26%

75 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-05 09:15 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for Pipeline. 

23""" 

24 

25import io 

26import re 

27import unittest 

28 

29import lsst.pipe.base.connectionTypes as cT 

30import lsst.utils.tests 

31from lsst.ctrl.mpexec.dotTools import pipeline2dot 

32from lsst.pipe.base import Pipeline, PipelineTask, PipelineTaskConfig, PipelineTaskConnections 

33 

34 

35class ExamplePipelineTaskConnections(PipelineTaskConnections, dimensions=()): 

36 input1 = cT.Input( 

37 name="", dimensions=["visit", "detector"], storageClass="example", doc="Input for this task" 

38 ) 

39 input2 = cT.Input( 

40 name="", dimensions=["visit", "detector"], storageClass="example", doc="Input for this task" 

41 ) 

42 output1 = cT.Output( 

43 name="", dimensions=["visit", "detector"], storageClass="example", doc="Output for this task" 

44 ) 

45 output2 = cT.Output( 

46 name="", dimensions=["visit", "detector"], storageClass="example", doc="Output for this task" 

47 ) 

48 

49 def __init__(self, *, config=None): 

50 super().__init__(config=config) 

51 if not config.connections.input2: 

52 self.inputs.remove("input2") 

53 if not config.connections.output2: 

54 self.outputs.remove("output2") 

55 

56 

57class ExamplePipelineTaskConfig(PipelineTaskConfig, pipelineConnections=ExamplePipelineTaskConnections): 

58 pass 

59 

60 

61def _makeConfig(inputName, outputName, pipeline, label): 

62 """Factory method for config instances 

63 

64 inputName and outputName can be either string or tuple of strings 

65 with two items max. 

66 """ 

67 if isinstance(inputName, tuple): 

68 pipeline.addConfigOverride(label, "connections.input1", inputName[0]) 

69 pipeline.addConfigOverride(label, "connections.input2", inputName[1] if len(inputName) > 1 else "") 

70 else: 

71 pipeline.addConfigOverride(label, "connections.input1", inputName) 

72 

73 if isinstance(outputName, tuple): 

74 pipeline.addConfigOverride(label, "connections.output1", outputName[0]) 

75 pipeline.addConfigOverride(label, "connections.output2", outputName[1] if len(outputName) > 1 else "") 

76 else: 

77 pipeline.addConfigOverride(label, "connections.output1", outputName) 

78 

79 

80class ExamplePipelineTask(PipelineTask): 

81 ConfigClass = ExamplePipelineTaskConfig 

82 

83 

84def _makePipeline(tasks): 

85 """Generate Pipeline instance. 

86 

87 Parameters 

88 ---------- 

89 tasks : list of tuples 

90 Each tuple in the list has 3 or 4 items: 

91 - input DatasetType name(s), string or tuple of strings 

92 - output DatasetType name(s), string or tuple of strings 

93 - task label, string 

94 - optional task class object, can be None 

95 

96 Returns 

97 ------- 

98 Pipeline instance 

99 """ 

100 pipe = Pipeline("test pipeline") 

101 for task in tasks: 

102 inputs = task[0] 

103 outputs = task[1] 

104 label = task[2] 

105 klass = task[3] if len(task) > 3 else ExamplePipelineTask 

106 pipe.addTask(klass, label) 

107 _makeConfig(inputs, outputs, pipe, label) 

108 return list(pipe.toExpandedPipeline()) 

109 

110 

111class DotToolsTestCase(unittest.TestCase): 

112 """A test case for dotTools""" 

113 

114 def testPipeline2dot(self): 

115 """Tests for dotTools.pipeline2dot method""" 

116 pipeline = _makePipeline( 

117 [ 

118 ("A", ("B", "C"), "task0"), 

119 ("C", "E", "task1"), 

120 ("B", "D", "task2"), 

121 (("D", "E"), "F", "task3"), 

122 ("D.C", "G", "task4"), 

123 ("task3_metadata", "H", "task5"), 

124 ] 

125 ) 

126 file = io.StringIO() 

127 pipeline2dot(pipeline, file) 

128 

129 # It's hard to validate complete output, just checking few basic 

130 # things, even that is not terribly stable. 

131 lines = file.getvalue().strip().split("\n") 

132 nglobals = 3 

133 ndatasets = 10 

134 ntasks = 6 

135 nedges = 16 

136 nextra = 2 # graph header and closing 

137 self.assertEqual(len(lines), nglobals + ndatasets + ntasks + nedges + nextra) 

138 

139 # make sure that all node names are quoted 

140 nodeRe = re.compile(r"^([^ ]+) \[.+\];$") 

141 edgeRe = re.compile(r"^([^ ]+) *-> *([^ ]+);$") 

142 for line in lines: 

143 match = nodeRe.match(line) 

144 if match: 

145 node = match.group(1) 

146 if node not in ["graph", "node", "edge"]: 

147 self.assertEqual(node[0] + node[-1], '""') 

148 continue 

149 match = edgeRe.match(line) 

150 if match: 

151 for group in (1, 2): 

152 node = match.group(group) 

153 self.assertEqual(node[0] + node[-1], '""') 

154 continue 

155 

156 # make sure components are connected appropriately 

157 self.assertIn('"D" -> "D.C"', file.getvalue()) 

158 

159 # make sure there is a connection created for metadata if someone 

160 # tries to read it in 

161 self.assertIn('"task3" -> "task3_metadata"', file.getvalue()) 

162 

163 

164class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

165 pass 

166 

167 

168def setup_module(module): 

169 lsst.utils.tests.init() 

170 

171 

172if __name__ == "__main__": 

173 lsst.utils.tests.init() 

174 unittest.main()