Coverage for tests/test_simple_pipeline_executor.py: 36%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import shutil
24import tempfile
25import unittest
27import lsst.daf.butler
28import lsst.utils.tests
29from lsst.ctrl.mpexec import SimplePipelineExecutor
30from lsst.pex.config import Field
31from lsst.pipe.base import PipelineTaskConfig, PipelineTaskConnections, TaskDef, connectionTypes
32from lsst.pipe.base.tests.no_dimensions import NoDimensionsTestTask
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class NoDimensionsTestConnections2(PipelineTaskConnections, dimensions=set()):
38 input = connectionTypes.Input(
39 name="input", doc="some dict-y input data for testing", storageClass="TaskMetadataLike"
40 )
41 output = connectionTypes.Output(
42 name="output", doc="some dict-y output data for testing", storageClass="StructuredDataDict"
43 )
46class NoDimensionsTestConfig2(PipelineTaskConfig, pipelineConnections=NoDimensionsTestConnections2):
47 key = Field(dtype=str, doc="String key for the dict entry the task sets.", default="one")
48 value = Field(dtype=int, doc="Integer value for the dict entry the task sets.", default=1)
49 outputSC = Field(dtype=str, doc="Output storage class requested", default="dict")
52class SimplePipelineExecutorTests(lsst.utils.tests.TestCase):
53 """Test the SimplePipelineExecutor API with a trivial task."""
55 def setUp(self):
56 self.path = tempfile.mkdtemp()
57 # standalone parameter forces the returned config to also include
58 # the information from the search paths.
59 config = lsst.daf.butler.Butler.makeRepo(
60 self.path, standalone=True, searchPaths=[os.path.join(TESTDIR, "config")]
61 )
62 self.butler = SimplePipelineExecutor.prep_butler(config, [], "fake")
63 self.butler.registry.registerDatasetType(
64 lsst.daf.butler.DatasetType(
65 "input",
66 dimensions=self.butler.registry.dimensions.empty,
67 storageClass="StructuredDataDict",
68 )
69 )
70 self.butler.put({"zero": 0}, "input")
72 def tearDown(self):
73 shutil.rmtree(self.path, ignore_errors=True)
75 def test_from_task_class(self):
76 """Test executing a single quantum with an executor created by the
77 `from_task_class` factory method, and the
78 `SimplePipelineExecutor.as_generator` method.
79 """
80 executor = SimplePipelineExecutor.from_task_class(NoDimensionsTestTask, butler=self.butler)
81 (quantum,) = executor.as_generator(register_dataset_types=True)
82 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1})
84 def _configure_pipeline(self, config_a_cls, config_b_cls, storageClass_a=None, storageClass_b=None):
85 """Configure a pipeline with from_pipeline."""
87 config_a = config_a_cls()
88 config_a.connections.output = "intermediate"
89 if storageClass_a:
90 config_a.outputSC = storageClass_a
91 config_b = config_b_cls()
92 config_b.connections.input = "intermediate"
93 if storageClass_b:
94 config_b.outputSC = storageClass_b
95 config_b.key = "two"
96 config_b.value = 2
97 task_defs = [
98 TaskDef(label="a", taskClass=NoDimensionsTestTask, config=config_a),
99 TaskDef(label="b", taskClass=NoDimensionsTestTask, config=config_b),
100 ]
101 executor = SimplePipelineExecutor.from_pipeline(task_defs, butler=self.butler)
102 return executor
104 def _test_logs(self, log_output, input_type_a, output_type_a, input_type_b, output_type_b):
105 """Check the expected input types received by tasks A and B"""
106 all_logs = "\n".join(log_output)
107 self.assertIn(f"lsst.a:Run method given data of type: {input_type_a}", all_logs)
108 self.assertIn(f"lsst.b:Run method given data of type: {input_type_b}", all_logs)
109 self.assertIn(f"lsst.a:Run method returns data of type: {output_type_a}", all_logs)
110 self.assertIn(f"lsst.b:Run method returns data of type: {output_type_b}", all_logs)
112 def test_from_pipeline(self):
113 """Test executing a two quanta from different configurations of the
114 same task, with an executor created by the `from_pipeline` factory
115 method, and the `SimplePipelineExecutor.run` method.
116 """
117 executor = self._configure_pipeline(
118 NoDimensionsTestTask.ConfigClass, NoDimensionsTestTask.ConfigClass
119 )
121 with self.assertLogs("lsst", level="INFO") as cm:
122 quanta = executor.run(register_dataset_types=True)
123 self._test_logs(cm.output, "dict", "dict", "dict", "dict")
125 self.assertEqual(len(quanta), 2)
126 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1})
127 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2})
129 def test_from_pipeline_intermediates_differ(self):
130 """Run pipeline but intermediates definition in registry differs."""
131 executor = self._configure_pipeline(
132 NoDimensionsTestTask.ConfigClass,
133 NoDimensionsTestTask.ConfigClass,
134 storageClass_b="TaskMetadataLike",
135 )
137 # Pre-define the "intermediate" storage class to be something that is
138 # like a dict but is not a dict. This will fail unless storage
139 # class conversion is supported in put and get.
140 self.butler.registry.registerDatasetType(
141 lsst.daf.butler.DatasetType(
142 "intermediate",
143 dimensions=self.butler.registry.dimensions.empty,
144 storageClass="TaskMetadataLike",
145 )
146 )
148 with self.assertLogs("lsst", level="INFO") as cm:
149 quanta = executor.run(register_dataset_types=True)
150 # A dict is given to task a without change.
151 # A returns a dict because it has not been told to do anything else.
152 # That does not match the storage class so it will be converted
153 # on put.
154 # b is given a TaskMetadata.
155 # b returns a TaskMetadata because that's how we configured it, but
156 # the butler expects a dict so it is converted on put.
157 self._test_logs(
158 cm.output, "dict", "dict", "lsst.pipe.base.TaskMetadata", "lsst.pipe.base.TaskMetadata"
159 )
161 self.assertEqual(len(quanta), 2)
162 self.assertEqual(self.butler.get("intermediate").to_dict(), {"zero": 0, "one": 1})
163 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2})
165 def test_from_pipeline_output_differ(self):
166 """Run pipeline but output definition in registry differs."""
167 executor = self._configure_pipeline(
168 NoDimensionsTestTask.ConfigClass,
169 NoDimensionsTestTask.ConfigClass,
170 storageClass_a="TaskMetadataLike",
171 )
173 # Pre-define the "output" storage class to be something that is
174 # like a dict but is not a dict. This will fail unless storage
175 # class conversion is supported in put and get.
176 self.butler.registry.registerDatasetType(
177 lsst.daf.butler.DatasetType(
178 "output",
179 dimensions=self.butler.registry.dimensions.empty,
180 storageClass="TaskMetadataLike",
181 )
182 )
184 with self.assertLogs("lsst", level="INFO") as cm:
185 quanta = executor.run(register_dataset_types=True)
186 # a has been told to return a TaskMetadata but will convert to dict.
187 # b returns a dict and that is converted to TaskMetadata on put.
188 self._test_logs(cm.output, "dict", "lsst.pipe.base.TaskMetadata", "dict", "dict")
190 self.assertEqual(len(quanta), 2)
191 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1})
192 self.assertEqual(self.butler.get("output").to_dict(), {"zero": 0, "one": 1, "two": 2})
194 def test_from_pipeline_input_differ(self):
195 """Run pipeline but input definition in registry differs."""
197 # This config declares that the pipeline takes a TaskMetadata
198 # as input but registry already thinks it has a StructureDataDict.
199 executor = self._configure_pipeline(NoDimensionsTestConfig2, NoDimensionsTestTask.ConfigClass)
201 with self.assertLogs("lsst", level="INFO") as cm:
202 quanta = executor.run(register_dataset_types=True)
203 self._test_logs(cm.output, "lsst.pipe.base.TaskMetadata", "dict", "dict", "dict")
205 self.assertEqual(len(quanta), 2)
206 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1})
207 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2})
209 def test_from_pipeline_incompatible(self):
210 """Run pipeline but definitions are not compatible."""
211 executor = self._configure_pipeline(
212 NoDimensionsTestTask.ConfigClass, NoDimensionsTestTask.ConfigClass
213 )
215 # Incompatible output dataset type.
216 self.butler.registry.registerDatasetType(
217 lsst.daf.butler.DatasetType(
218 "output",
219 dimensions=self.butler.registry.dimensions.empty,
220 storageClass="StructuredDataList",
221 )
222 )
224 with self.assertRaisesRegex(
225 ValueError, "StructuredDataDict.*inconsistent with registry definition.*StructuredDataList"
226 ):
227 executor.run(register_dataset_types=True)
229 def test_from_pipeline_file(self):
230 """Test executing a two quanta from different configurations of the
231 same task, with an executor created by the `from_pipeline_filename`
232 factory method, and the `SimplePipelineExecutor.run` method.
233 """
234 filename = os.path.join(self.path, "pipeline.yaml")
235 with open(filename, "w") as f:
236 f.write(
237 """
238 description: test
239 tasks:
240 a:
241 class: "lsst.pipe.base.tests.no_dimensions.NoDimensionsTestTask"
242 config:
243 connections.output: "intermediate"
244 b:
245 class: "lsst.pipe.base.tests.no_dimensions.NoDimensionsTestTask"
246 config:
247 connections.input: "intermediate"
248 key: "two"
249 value: 2
250 """
251 )
252 executor = SimplePipelineExecutor.from_pipeline_filename(filename, butler=self.butler)
253 quanta = executor.run(register_dataset_types=True)
254 self.assertEqual(len(quanta), 2)
255 self.assertEqual(self.butler.get("intermediate"), {"zero": 0, "one": 1})
256 self.assertEqual(self.butler.get("output"), {"zero": 0, "one": 1, "two": 2})
259class MemoryTester(lsst.utils.tests.MemoryTestCase):
260 pass
263def setup_module(module):
264 lsst.utils.tests.init()
267if __name__ == "__main__": 267 ↛ 268line 267 didn't jump to line 268, because the condition on line 267 was never true
268 lsst.utils.tests.init()
269 unittest.main()