Coverage for tests/test_simple_pipeline_executor.py: 36%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

106 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import tempfile 

25import unittest 

26 

27import lsst.daf.butler 

28import lsst.utils.tests 

29from lsst.ctrl.mpexec import SimplePipelineExecutor 

30from lsst.pex.config import Field 

31from lsst.pipe.base import PipelineTaskConfig, PipelineTaskConnections, TaskDef, connectionTypes 

32from lsst.pipe.base.tests.no_dimensions import NoDimensionsTestTask 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class NoDimensionsTestConnections2(PipelineTaskConnections, dimensions=set()): 

38 input = connectionTypes.Input( 

39 name="input", doc="some dict-y input data for testing", storageClass="TaskMetadataLike" 

40 ) 

41 output = connectionTypes.Output( 

42 name="output", doc="some dict-y output data for testing", storageClass="StructuredDataDict" 

43 ) 

44 

45 

46class NoDimensionsTestConfig2(PipelineTaskConfig, pipelineConnections=NoDimensionsTestConnections2): 

47 key = Field(dtype=str, doc="String key for the dict entry the task sets.", default="one") 

48 value = Field(dtype=int, doc="Integer value for the dict entry the task sets.", default=1) 

49 outputSC = Field(dtype=str, doc="Output storage class requested", default="dict") 

50 

51 

52class SimplePipelineExecutorTests(lsst.utils.tests.TestCase): 

53 """Test the SimplePipelineExecutor API with a trivial task.""" 

54 

55 def setUp(self): 

56 self.path = tempfile.mkdtemp() 

57 # standalone parameter forces the returned config to also include 

58 # the information from the search paths. 

59 config = lsst.daf.butler.Butler.makeRepo( 

60 self.path, standalone=True, searchPaths=[os.path.join(TESTDIR, "config")] 

61 ) 

62 self.butler = SimplePipelineExecutor.prep_butler(config, [], "fake") 

63 self.butler.registry.registerDatasetType( 

64 lsst.daf.butler.DatasetType( 

65 "input", 

66 dimensions=self.butler.registry.dimensions.empty, 

67 storageClass="StructuredDataDict", 

68 ) 

69 ) 

70 self.butler.put({"zero": 0}, "input") 

71 

72 def tearDown(self): 

73 shutil.rmtree(self.path, ignore_errors=True) 

74 

75 def test_from_task_class(self): 

76 """Test executing a single quantum with an executor created by the 

77 `from_task_class` factory method, and the 

78 `SimplePipelineExecutor.as_generator` method. 

79 """ 

80 executor = SimplePipelineExecutor.from_task_class(NoDimensionsTestTask, butler=self.butler) 

81 (quantum,) = executor.as_generator(register_dataset_types=True) 

82 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1}) 

83 

84 def _configure_pipeline(self, config_a_cls, config_b_cls, storageClass_a=None, storageClass_b=None): 

85 """Configure a pipeline with from_pipeline.""" 

86 

87 config_a = config_a_cls() 

88 config_a.connections.output = "intermediate" 

89 if storageClass_a: 

90 config_a.outputSC = storageClass_a 

91 config_b = config_b_cls() 

92 config_b.connections.input = "intermediate" 

93 if storageClass_b: 

94 config_b.outputSC = storageClass_b 

95 config_b.key = "two" 

96 config_b.value = 2 

97 task_defs = [ 

98 TaskDef(label="a", taskClass=NoDimensionsTestTask, config=config_a), 

99 TaskDef(label="b", taskClass=NoDimensionsTestTask, config=config_b), 

100 ] 

101 executor = SimplePipelineExecutor.from_pipeline(task_defs, butler=self.butler) 

102 return executor 

103 

104 def _test_logs(self, log_output, input_type_a, output_type_a, input_type_b, output_type_b): 

105 """Check the expected input types received by tasks A and B""" 

106 all_logs = "\n".join(log_output) 

107 self.assertIn(f"lsst.a:Run method given data of type: {input_type_a}", all_logs) 

108 self.assertIn(f"lsst.b:Run method given data of type: {input_type_b}", all_logs) 

109 self.assertIn(f"lsst.a:Run method returns data of type: {output_type_a}", all_logs) 

110 self.assertIn(f"lsst.b:Run method returns data of type: {output_type_b}", all_logs) 

111 

112 def test_from_pipeline(self): 

113 """Test executing a two quanta from different configurations of the 

114 same task, with an executor created by the `from_pipeline` factory 

115 method, and the `SimplePipelineExecutor.run` method. 

116 """ 

117 executor = self._configure_pipeline( 

118 NoDimensionsTestTask.ConfigClass, NoDimensionsTestTask.ConfigClass 

119 ) 

120 

121 with self.assertLogs("lsst", level="INFO") as cm: 

122 quanta = executor.run(register_dataset_types=True) 

123 self._test_logs(cm.output, "dict", "dict", "dict", "dict") 

124 

125 self.assertEqual(len(quanta), 2) 

126 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1}) 

127 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2}) 

128 

129 def test_from_pipeline_intermediates_differ(self): 

130 """Run pipeline but intermediates definition in registry differs.""" 

131 executor = self._configure_pipeline( 

132 NoDimensionsTestTask.ConfigClass, 

133 NoDimensionsTestTask.ConfigClass, 

134 storageClass_b="TaskMetadataLike", 

135 ) 

136 

137 # Pre-define the "intermediate" storage class to be something that is 

138 # like a dict but is not a dict. This will fail unless storage 

139 # class conversion is supported in put and get. 

140 self.butler.registry.registerDatasetType( 

141 lsst.daf.butler.DatasetType( 

142 "intermediate", 

143 dimensions=self.butler.registry.dimensions.empty, 

144 storageClass="TaskMetadataLike", 

145 ) 

146 ) 

147 

148 with self.assertLogs("lsst", level="INFO") as cm: 

149 quanta = executor.run(register_dataset_types=True) 

150 # A dict is given to task a without change. 

151 # A returns a dict because it has not been told to do anything else. 

152 # That does not match the storage class so it will be converted 

153 # on put. 

154 # b is given a TaskMetadata. 

155 # b returns a TaskMetadata because that's how we configured it, but 

156 # the butler expects a dict so it is converted on put. 

157 self._test_logs( 

158 cm.output, "dict", "dict", "lsst.pipe.base.TaskMetadata", "lsst.pipe.base.TaskMetadata" 

159 ) 

160 

161 self.assertEqual(len(quanta), 2) 

162 self.assertEqual(self.butler.get("intermediate").to_dict(), {"zero": 0, "one": 1}) 

163 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2}) 

164 

165 def test_from_pipeline_output_differ(self): 

166 """Run pipeline but output definition in registry differs.""" 

167 executor = self._configure_pipeline( 

168 NoDimensionsTestTask.ConfigClass, 

169 NoDimensionsTestTask.ConfigClass, 

170 storageClass_a="TaskMetadataLike", 

171 ) 

172 

173 # Pre-define the "output" storage class to be something that is 

174 # like a dict but is not a dict. This will fail unless storage 

175 # class conversion is supported in put and get. 

176 self.butler.registry.registerDatasetType( 

177 lsst.daf.butler.DatasetType( 

178 "output", 

179 dimensions=self.butler.registry.dimensions.empty, 

180 storageClass="TaskMetadataLike", 

181 ) 

182 ) 

183 

184 with self.assertLogs("lsst", level="INFO") as cm: 

185 quanta = executor.run(register_dataset_types=True) 

186 # a has been told to return a TaskMetadata but will convert to dict. 

187 # b returns a dict and that is converted to TaskMetadata on put. 

188 self._test_logs(cm.output, "dict", "lsst.pipe.base.TaskMetadata", "dict", "dict") 

189 

190 self.assertEqual(len(quanta), 2) 

191 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1}) 

192 self.assertEqual(self.butler.get("output").to_dict(), {"zero": 0, "one": 1, "two": 2}) 

193 

194 def test_from_pipeline_input_differ(self): 

195 """Run pipeline but input definition in registry differs.""" 

196 

197 # This config declares that the pipeline takes a TaskMetadata 

198 # as input but registry already thinks it has a StructureDataDict. 

199 executor = self._configure_pipeline(NoDimensionsTestConfig2, NoDimensionsTestTask.ConfigClass) 

200 

201 with self.assertLogs("lsst", level="INFO") as cm: 

202 quanta = executor.run(register_dataset_types=True) 

203 self._test_logs(cm.output, "lsst.pipe.base.TaskMetadata", "dict", "dict", "dict") 

204 

205 self.assertEqual(len(quanta), 2) 

206 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1}) 

207 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2}) 

208 

209 def test_from_pipeline_incompatible(self): 

210 """Run pipeline but definitions are not compatible.""" 

211 executor = self._configure_pipeline( 

212 NoDimensionsTestTask.ConfigClass, NoDimensionsTestTask.ConfigClass 

213 ) 

214 

215 # Incompatible output dataset type. 

216 self.butler.registry.registerDatasetType( 

217 lsst.daf.butler.DatasetType( 

218 "output", 

219 dimensions=self.butler.registry.dimensions.empty, 

220 storageClass="StructuredDataList", 

221 ) 

222 ) 

223 

224 with self.assertRaisesRegex( 

225 ValueError, "StructuredDataDict.*inconsistent with registry definition.*StructuredDataList" 

226 ): 

227 executor.run(register_dataset_types=True) 

228 

229 def test_from_pipeline_file(self): 

230 """Test executing a two quanta from different configurations of the 

231 same task, with an executor created by the `from_pipeline_filename` 

232 factory method, and the `SimplePipelineExecutor.run` method. 

233 """ 

234 filename = os.path.join(self.path, "pipeline.yaml") 

235 with open(filename, "w") as f: 

236 f.write( 

237 """ 

238 description: test 

239 tasks: 

240 a: 

241 class: "lsst.pipe.base.tests.no_dimensions.NoDimensionsTestTask" 

242 config: 

243 connections.output: "intermediate" 

244 b: 

245 class: "lsst.pipe.base.tests.no_dimensions.NoDimensionsTestTask" 

246 config: 

247 connections.input: "intermediate" 

248 key: "two" 

249 value: 2 

250 """ 

251 ) 

252 executor = SimplePipelineExecutor.from_pipeline_filename(filename, butler=self.butler) 

253 quanta = executor.run(register_dataset_types=True) 

254 self.assertEqual(len(quanta), 2) 

255 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1}) 

256 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2}) 

257 

258 

259class MemoryTester(lsst.utils.tests.MemoryTestCase): 

260 pass 

261 

262 

263def setup_module(module): 

264 lsst.utils.tests.init() 

265 

266 

267if __name__ == "__main__": 267 ↛ 268line 267 didn't jump to line 268, because the condition on line 267 was never true

268 lsst.utils.tests.init() 

269 unittest.main()