Coverage for tests/test_pipelineIR.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2# This file is part of pipe_base.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
23import os
24import tempfile
25import textwrap
26import unittest
28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR
29import lsst.utils.tests
32class ConfigIRTestCase(unittest.TestCase):
33 """A test case for ConfigIR Objects
35 ConfigIR contains a method that is not exercised by the PipelineIR task,
36 so it should be tested here
37 """
39 def setUp(self):
40 pass
42 def tearDown(self):
43 pass
45 def testMergeConfig(self):
46 # Create some configs to merge
47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"],
48 rest={"a": 1, "b": 2})
49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2})
50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2})
51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4})
52 config5 = ConfigIR(rest={"f": 5, "g": 6})
53 config6 = ConfigIR(rest={"h": 7, "i": 8})
54 config7 = ConfigIR(rest={"h": 9})
56 # Merge configs with different dataIds, this should yield two elements
57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2])
59 # Merge configs with python blocks defined, this should yield two elements
60 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3])
62 # Merge configs with file defined, this should yield two elements
63 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4])
65 # merge config2 into config1
66 merge_result = list(config5.maybe_merge(config6))
67 self.assertEqual(len(merge_result), 1)
68 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8})
70 # Cant merge configs with shared keys
71 self.assertEqual(list(config5.maybe_merge(config7)), [config5, config7])
74class PipelineIRTestCase(unittest.TestCase):
75 """A test case for PipelineIR objects
76 """
78 def setUp(self):
79 pass
81 def tearDown(self):
82 pass
84 def testPipelineIRInitChecks(self):
85 # Missing description
86 pipeline_str = """
87 tasks:
88 a: module.A
89 """
90 with self.assertRaises(ValueError):
91 PipelineIR.from_string(pipeline_str)
93 # Missing tasks
94 pipeline_str = """
95 description: Test Pipeline
96 """
97 with self.assertRaises(ValueError):
98 PipelineIR.from_string(pipeline_str)
100 # This should raise a FileNotFoundError, as there are inherits defined
101 # so the __init__ method should pass but the inherited file does not
102 # exist
103 pipeline_str = textwrap.dedent("""
104 description: Test Pipeline
105 inherits: /dummy_pipeline.yaml
106 """)
108 with self.assertRaises(FileNotFoundError):
109 PipelineIR.from_string(pipeline_str)
111 def testTaskParsing(self):
112 # Should be able to parse a task defined both ways
113 pipeline_str = textwrap.dedent("""
114 description: Test Pipeline
115 tasks:
116 modA: test.modA
117 modB:
118 class: test.modB
119 """)
121 pipeline = PipelineIR.from_string(pipeline_str)
122 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"])
123 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"])
125 def testInheritParsing(self):
126 # This should raise, as the two pipelines, both define the same label
127 pipeline_str = textwrap.dedent("""
128 description: Test Pipeline
129 inherits:
130 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
131 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
132 """)
134 with self.assertRaises(ValueError):
135 PipelineIR.from_string(pipeline_str)
137 # This should pass, as the conflicting task is excluded
138 pipeline_str = textwrap.dedent("""
139 description: Test Pipeline
140 inherits:
141 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
142 exclude: modA
143 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
144 """)
145 pipeline = PipelineIR.from_string(pipeline_str)
146 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
148 # This should pass, as the conflicting task is no in includes
149 pipeline_str = textwrap.dedent("""
150 description: Test Pipeline
151 inherits:
152 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
153 include: modB
154 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
155 """)
157 pipeline = PipelineIR.from_string(pipeline_str)
158 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
160 # Test that you cant include and exclude a task
161 pipeline_str = textwrap.dedent("""
162 description: Test Pipeline
163 inherits:
164 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
165 exclude: modA
166 include: modB
167 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
168 """)
170 with self.assertRaises(ValueError):
171 PipelineIR.from_string(pipeline_str)
173 # Test that contracts are inherited
174 pipeline_str = textwrap.dedent("""
175 description: Test Pipeline
176 inherits:
177 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
178 """)
180 pipeline = PipelineIR.from_string(pipeline_str)
181 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
183 # Test that contracts are not inherited
184 pipeline_str = textwrap.dedent("""
185 description: Test Pipeline
186 inherits:
187 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
188 importContracts: False
189 """)
191 pipeline = PipelineIR.from_string(pipeline_str)
192 self.assertEqual(pipeline.contracts, [])
194 def testReadContracts(self):
195 # Verify that contracts are read in from a pipeline
196 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml")
197 pipeline = PipelineIR.from_file(location)
198 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
200 # Verify that a contract message is loaded
201 pipeline_str = textwrap.dedent("""
202 description: Test Pipeline
203 tasks:
204 modA: test.modA
205 modB:
206 class: test.modB
207 contracts:
208 - contract: modA.foo == modB.Bar
209 msg: "Test message"
210 """)
212 pipeline = PipelineIR.from_string(pipeline_str)
213 self.assertEqual(pipeline.contracts[0].msg, "Test message")
215 def testInstrument(self):
216 # Verify that if instrument is defined it is parsed out
217 pipeline_str = textwrap.dedent("""
218 description: Test Pipeline
219 instrument: dummyCam
220 tasks:
221 modA: test.moduleA
222 """)
224 pipeline = PipelineIR.from_string(pipeline_str)
225 self.assertEqual(pipeline.instrument, "dummyCam")
227 def testReadTaskConfig(self):
228 # Verify that a task with a config is read in correctly
229 pipeline_str = textwrap.dedent("""
230 description: Test Pipeline
231 tasks:
232 modA:
233 class: test.moduleA
234 config:
235 propertyA: 6
236 propertyB: 7
237 file: testfile.py
238 python: "config.testDict['a'] = 9"
239 """)
241 pipeline = PipelineIR.from_string(pipeline_str)
242 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"])
243 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9")
244 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
246 # Verify that multiple files are read fine
247 pipeline_str = textwrap.dedent("""
248 description: Test Pipeline
249 tasks:
250 modA:
251 class: test.moduleA
252 config:
253 file:
254 - testfile.py
255 - otherFile.py
256 """)
258 pipeline = PipelineIR.from_string(pipeline_str)
259 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"])
261 # Test reading multiple Config entries
262 pipeline_str = textwrap.dedent("""
263 description: Test Pipeline
264 tasks:
265 modA:
266 class: test.moduleA
267 config:
268 - propertyA: 6
269 propertyB: 7
270 dataId: {"visit": 6}
271 - propertyA: 8
272 propertyB: 9
273 """)
275 pipeline = PipelineIR.from_string(pipeline_str)
276 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
277 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6})
278 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9})
279 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None)
281 def testSerialization(self):
282 # Test creating a pipeline, writing it to a file, reading the file
283 pipeline_str = textwrap.dedent("""
284 description: Test Pipeline
285 instrument: dummyCam
286 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml
287 tasks:
288 modC:
289 class: test.moduleC
290 config:
291 - propertyA: 6
292 propertyB: 7
293 dataId: {"visit": 6}
294 - propertyA: 8
295 propertyB: 9
296 modD: test.moduleD
297 contracts:
298 - modA.foo == modB.bar
299 """)
301 pipeline = PipelineIR.from_string(pipeline_str)
303 # Create the temp file, write and read
304 with tempfile.NamedTemporaryFile() as tf:
305 pipeline.to_file(tf.name)
306 loaded_pipeline = PipelineIR.from_file(tf.name)
307 self.assertEqual(pipeline, loaded_pipeline)
309 def testPipelineYamlLoader(self):
310 # Tests that an exception is thrown in the case a key is used multiple
311 # times in a given scope within a pipeline file
312 pipeline_str = textwrap.dedent("""
313 description: Test Pipeline
314 tasks:
315 modA: test1
316 modB: test2
317 modA: test3
318 """)
319 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str)
322class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
323 pass
326def setup_module(module):
327 lsst.utils.tests.init()
330if __name__ == "__main__": 330 ↛ 331line 330 didn't jump to line 331, because the condition on line 330 was never true
331 lsst.utils.tests.init()
332 unittest.main()