Coverage for tests/test_pipelineIR.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2# This file is part of pipe_base.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
23import os
24import tempfile
25import textwrap
26import unittest
28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR
29import lsst.utils.tests
32class ConfigIRTestCase(unittest.TestCase):
33 """A test case for ConfigIR Objects
35 ConfigIR contains a method that is not exercised by the PipelineIR task,
36 so it should be tested here
37 """
39 def setUp(self):
40 pass
42 def tearDown(self):
43 pass
45 def testMergeConfig(self):
46 # Create some configs to merge
47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"],
48 rest={"a": 1, "b": 2})
49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2})
50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2})
51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4})
52 config5 = ConfigIR(rest={"f": 5, "g": 6})
53 config6 = ConfigIR(rest={"h": 7, "i": 8})
54 config7 = ConfigIR(rest={"h": 9})
56 # Merge configs with different dataIds, this should yield two elements
57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2])
59 # Merge configs with python blocks defined, this should yield two elements
60 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3])
62 # Merge configs with file defined, this should yield two elements
63 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4])
65 # merge config2 into config1
66 merge_result = list(config5.maybe_merge(config6))
67 self.assertEqual(len(merge_result), 1)
68 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8})
70 # Cant merge configs with shared keys
71 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7])
74class PipelineIRTestCase(unittest.TestCase):
75 """A test case for PipelineIR objects
76 """
78 def setUp(self):
79 pass
81 def tearDown(self):
82 pass
84 def testPipelineIRInitChecks(self):
85 # Missing description
86 pipeline_str = """
87 tasks:
88 a: module.A
89 """
90 with self.assertRaises(ValueError):
91 PipelineIR.from_string(pipeline_str)
93 # Missing tasks
94 pipeline_str = """
95 description: Test Pipeline
96 """
97 with self.assertRaises(ValueError):
98 PipelineIR.from_string(pipeline_str)
100 # This should raise a FileNotFoundError, as there are inherits defined
101 # so the __init__ method should pass but the inherited file does not
102 # exist
103 pipeline_str = textwrap.dedent("""
104 description: Test Pipeline
105 inherits: /dummy_pipeline.yaml
106 """)
108 with self.assertRaises(FileNotFoundError):
109 PipelineIR.from_string(pipeline_str)
111 def testTaskParsing(self):
112 # Should be able to parse a task defined both ways
113 pipeline_str = textwrap.dedent("""
114 description: Test Pipeline
115 tasks:
116 modA: test.modA
117 modB:
118 class: test.modB
119 """)
121 pipeline = PipelineIR.from_string(pipeline_str)
122 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"])
123 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"])
125 def testInheritParsing(self):
126 # This should raise, as the two pipelines, both define the same label
127 pipeline_str = textwrap.dedent("""
128 description: Test Pipeline
129 inherits:
130 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
131 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
132 """)
134 with self.assertRaises(ValueError):
135 PipelineIR.from_string(pipeline_str)
137 # This should pass, as the conflicting task is excluded
138 pipeline_str = textwrap.dedent("""
139 description: Test Pipeline
140 inherits:
141 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
142 exclude: modA
143 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
144 """)
145 pipeline = PipelineIR.from_string(pipeline_str)
146 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
148 # This should pass, as the conflicting task is no in includes
149 pipeline_str = textwrap.dedent("""
150 description: Test Pipeline
151 inherits:
152 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
153 include: modB
154 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
155 """)
157 pipeline = PipelineIR.from_string(pipeline_str)
158 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
160 # Test that you cant include and exclude a task
161 pipeline_str = textwrap.dedent("""
162 description: Test Pipeline
163 inherits:
164 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
165 exclude: modA
166 include: modB
167 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
168 """)
170 with self.assertRaises(ValueError):
171 PipelineIR.from_string(pipeline_str)
173 # Test that contracts are inherited
174 pipeline_str = textwrap.dedent("""
175 description: Test Pipeline
176 inherits:
177 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
178 """)
180 pipeline = PipelineIR.from_string(pipeline_str)
181 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
183 # Test that contracts are not inherited
184 pipeline_str = textwrap.dedent("""
185 description: Test Pipeline
186 inherits:
187 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
188 importContracts: False
189 """)
191 pipeline = PipelineIR.from_string(pipeline_str)
192 self.assertEqual(pipeline.contracts, [])
194 # Test that configs are inherited when defining the same task again with
195 # the same label
196 pipeline_str = textwrap.dedent("""
197 description: Test Pipeline
198 inherits:
199 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
200 tasks:
201 modA:
202 class: "test.moduleA"
203 config:
204 value2: 2
205 """)
206 pipeline = PipelineIR.from_string(pipeline_str)
207 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2})
209 # Test that configs are not inherited when redefining the task
210 # associated with a label
211 pipeline_str = textwrap.dedent("""
212 description: Test Pipeline
213 inherits:
214 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
215 tasks:
216 modA:
217 class: "test.moduleAReplace"
218 config:
219 value2: 2
220 """)
221 pipeline = PipelineIR.from_string(pipeline_str)
222 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2})
224 def testReadContracts(self):
225 # Verify that contracts are read in from a pipeline
226 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml")
227 pipeline = PipelineIR.from_file(location)
228 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
230 # Verify that a contract message is loaded
231 pipeline_str = textwrap.dedent("""
232 description: Test Pipeline
233 tasks:
234 modA: test.modA
235 modB:
236 class: test.modB
237 contracts:
238 - contract: modA.foo == modB.Bar
239 msg: "Test message"
240 """)
242 pipeline = PipelineIR.from_string(pipeline_str)
243 self.assertEqual(pipeline.contracts[0].msg, "Test message")
245 def testInstrument(self):
246 # Verify that if instrument is defined it is parsed out
247 pipeline_str = textwrap.dedent("""
248 description: Test Pipeline
249 instrument: dummyCam
250 tasks:
251 modA: test.moduleA
252 """)
254 pipeline = PipelineIR.from_string(pipeline_str)
255 self.assertEqual(pipeline.instrument, "dummyCam")
257 def testReadTaskConfig(self):
258 # Verify that a task with a config is read in correctly
259 pipeline_str = textwrap.dedent("""
260 description: Test Pipeline
261 tasks:
262 modA:
263 class: test.moduleA
264 config:
265 propertyA: 6
266 propertyB: 7
267 file: testfile.py
268 python: "config.testDict['a'] = 9"
269 """)
271 pipeline = PipelineIR.from_string(pipeline_str)
272 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"])
273 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9")
274 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
276 # Verify that multiple files are read fine
277 pipeline_str = textwrap.dedent("""
278 description: Test Pipeline
279 tasks:
280 modA:
281 class: test.moduleA
282 config:
283 file:
284 - testfile.py
285 - otherFile.py
286 """)
288 pipeline = PipelineIR.from_string(pipeline_str)
289 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"])
291 # Test reading multiple Config entries
292 pipeline_str = textwrap.dedent("""
293 description: Test Pipeline
294 tasks:
295 modA:
296 class: test.moduleA
297 config:
298 - propertyA: 6
299 propertyB: 7
300 dataId: {"visit": 6}
301 - propertyA: 8
302 propertyB: 9
303 """)
305 pipeline = PipelineIR.from_string(pipeline_str)
306 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
307 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6})
308 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9})
309 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None)
311 def testSerialization(self):
312 # Test creating a pipeline, writing it to a file, reading the file
313 pipeline_str = textwrap.dedent("""
314 description: Test Pipeline
315 instrument: dummyCam
316 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml
317 tasks:
318 modC:
319 class: test.moduleC
320 config:
321 - propertyA: 6
322 propertyB: 7
323 dataId: {"visit": 6}
324 - propertyA: 8
325 propertyB: 9
326 modD: test.moduleD
327 contracts:
328 - modA.foo == modB.bar
329 """)
331 pipeline = PipelineIR.from_string(pipeline_str)
333 # Create the temp file, write and read
334 with tempfile.NamedTemporaryFile() as tf:
335 pipeline.to_file(tf.name)
336 loaded_pipeline = PipelineIR.from_file(tf.name)
337 self.assertEqual(pipeline, loaded_pipeline)
339 def testPipelineYamlLoader(self):
340 # Tests that an exception is thrown in the case a key is used multiple
341 # times in a given scope within a pipeline file
342 pipeline_str = textwrap.dedent("""
343 description: Test Pipeline
344 tasks:
345 modA: test1
346 modB: test2
347 modA: test3
348 """)
349 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str)
352class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
353 pass
356def setup_module(module):
357 lsst.utils.tests.init()
360if __name__ == "__main__": 360 ↛ 361line 360 didn't jump to line 361, because the condition on line 360 was never true
361 lsst.utils.tests.init()
362 unittest.main()