Coverage for tests/test_pipelineIR.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2# This file is part of pipe_base.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
23import os
24import tempfile
25import textwrap
26import unittest
28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR
29import lsst.utils.tests
32class ConfigIRTestCase(unittest.TestCase):
33 """A test case for ConfigIR Objects
35 ConfigIR contains a method that is not exercised by the PipelineIR task,
36 so it should be tested here
37 """
39 def setUp(self):
40 pass
42 def tearDown(self):
43 pass
45 def testMergeConfig(self):
46 # Create some configs to merge
47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"],
48 rest={"a": 1, "b": 2})
49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2})
50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2})
51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4})
52 config5 = ConfigIR(rest={"f": 5, "g": 6})
53 config6 = ConfigIR(rest={"h": 7, "i": 8})
54 config7 = ConfigIR(rest={"h": 9})
56 # Merge configs with different dataIds, this should yield two elements
57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2])
59 # Merge configs with python blocks defined, this should yield two
60 # elements
61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3])
63 # Merge configs with file defined, this should yield two elements
64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4])
66 # merge config2 into config1
67 merge_result = list(config5.maybe_merge(config6))
68 self.assertEqual(len(merge_result), 1)
69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8})
71 # Cant merge configs with shared keys
72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7])
75class PipelineIRTestCase(unittest.TestCase):
76 """A test case for PipelineIR objects
77 """
79 def setUp(self):
80 pass
82 def tearDown(self):
83 pass
85 def testPipelineIRInitChecks(self):
86 # Missing description
87 pipeline_str = """
88 tasks:
89 a: module.A
90 """
91 with self.assertRaises(ValueError):
92 PipelineIR.from_string(pipeline_str)
94 # Missing tasks
95 pipeline_str = """
96 description: Test Pipeline
97 """
98 with self.assertRaises(ValueError):
99 PipelineIR.from_string(pipeline_str)
101 # This should raise a FileNotFoundError, as there are inherits defined
102 # so the __init__ method should pass but the inherited file does not
103 # exist
104 pipeline_str = textwrap.dedent("""
105 description: Test Pipeline
106 inherits: /dummy_pipeline.yaml
107 """)
109 with self.assertRaises(FileNotFoundError):
110 PipelineIR.from_string(pipeline_str)
112 def testTaskParsing(self):
113 # Should be able to parse a task defined both ways
114 pipeline_str = textwrap.dedent("""
115 description: Test Pipeline
116 tasks:
117 modA: test.modA
118 modB:
119 class: test.modB
120 """)
122 pipeline = PipelineIR.from_string(pipeline_str)
123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"])
124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"])
126 def testInheritParsing(self):
127 # This should raise, as the two pipelines, both define the same label
128 pipeline_str = textwrap.dedent("""
129 description: Test Pipeline
130 inherits:
131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
133 """)
135 with self.assertRaises(ValueError):
136 PipelineIR.from_string(pipeline_str)
138 # This should pass, as the conflicting task is excluded
139 pipeline_str = textwrap.dedent("""
140 description: Test Pipeline
141 inherits:
142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
143 exclude: modA
144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
145 """)
146 pipeline = PipelineIR.from_string(pipeline_str)
147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
149 # This should pass, as the conflicting task is no in includes
150 pipeline_str = textwrap.dedent("""
151 description: Test Pipeline
152 inherits:
153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
154 include: modB
155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
156 """)
158 pipeline = PipelineIR.from_string(pipeline_str)
159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
161 # Test that you cant include and exclude a task
162 pipeline_str = textwrap.dedent("""
163 description: Test Pipeline
164 inherits:
165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
166 exclude: modA
167 include: modB
168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
169 """)
171 with self.assertRaises(ValueError):
172 PipelineIR.from_string(pipeline_str)
174 # Test that contracts are inherited
175 pipeline_str = textwrap.dedent("""
176 description: Test Pipeline
177 inherits:
178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
179 """)
181 pipeline = PipelineIR.from_string(pipeline_str)
182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
184 # Test that contracts are not inherited
185 pipeline_str = textwrap.dedent("""
186 description: Test Pipeline
187 inherits:
188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
189 importContracts: False
190 """)
192 pipeline = PipelineIR.from_string(pipeline_str)
193 self.assertEqual(pipeline.contracts, [])
195 # Test that configs are inherited when defining the same task again
196 # with the same label
197 pipeline_str = textwrap.dedent("""
198 description: Test Pipeline
199 inherits:
200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
201 tasks:
202 modA:
203 class: "test.moduleA"
204 config:
205 value2: 2
206 """)
207 pipeline = PipelineIR.from_string(pipeline_str)
208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2})
210 # Test that configs are not inherited when redefining the task
211 # associated with a label
212 pipeline_str = textwrap.dedent("""
213 description: Test Pipeline
214 inherits:
215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
216 tasks:
217 modA:
218 class: "test.moduleAReplace"
219 config:
220 value2: 2
221 """)
222 pipeline = PipelineIR.from_string(pipeline_str)
223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2})
225 def testReadContracts(self):
226 # Verify that contracts are read in from a pipeline
227 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml")
228 pipeline = PipelineIR.from_file(location)
229 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
231 # Verify that a contract message is loaded
232 pipeline_str = textwrap.dedent("""
233 description: Test Pipeline
234 tasks:
235 modA: test.modA
236 modB:
237 class: test.modB
238 contracts:
239 - contract: modA.foo == modB.Bar
240 msg: "Test message"
241 """)
243 pipeline = PipelineIR.from_string(pipeline_str)
244 self.assertEqual(pipeline.contracts[0].msg, "Test message")
246 def testInstrument(self):
247 # Verify that if instrument is defined it is parsed out
248 pipeline_str = textwrap.dedent("""
249 description: Test Pipeline
250 instrument: dummyCam
251 tasks:
252 modA: test.moduleA
253 """)
255 pipeline = PipelineIR.from_string(pipeline_str)
256 self.assertEqual(pipeline.instrument, "dummyCam")
258 def testReadTaskConfig(self):
259 # Verify that a task with a config is read in correctly
260 pipeline_str = textwrap.dedent("""
261 description: Test Pipeline
262 tasks:
263 modA:
264 class: test.moduleA
265 config:
266 propertyA: 6
267 propertyB: 7
268 file: testfile.py
269 python: "config.testDict['a'] = 9"
270 """)
272 pipeline = PipelineIR.from_string(pipeline_str)
273 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"])
274 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9")
275 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
277 # Verify that multiple files are read fine
278 pipeline_str = textwrap.dedent("""
279 description: Test Pipeline
280 tasks:
281 modA:
282 class: test.moduleA
283 config:
284 file:
285 - testfile.py
286 - otherFile.py
287 """)
289 pipeline = PipelineIR.from_string(pipeline_str)
290 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"])
292 # Test reading multiple Config entries
293 pipeline_str = textwrap.dedent("""
294 description: Test Pipeline
295 tasks:
296 modA:
297 class: test.moduleA
298 config:
299 - propertyA: 6
300 propertyB: 7
301 dataId: {"visit": 6}
302 - propertyA: 8
303 propertyB: 9
304 """)
306 pipeline = PipelineIR.from_string(pipeline_str)
307 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
308 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6})
309 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9})
310 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None)
312 def testSerialization(self):
313 # Test creating a pipeline, writing it to a file, reading the file
314 pipeline_str = textwrap.dedent("""
315 description: Test Pipeline
316 instrument: dummyCam
317 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml
318 tasks:
319 modC:
320 class: test.moduleC
321 config:
322 - propertyA: 6
323 propertyB: 7
324 dataId: {"visit": 6}
325 - propertyA: 8
326 propertyB: 9
327 modD: test.moduleD
328 contracts:
329 - modA.foo == modB.bar
330 """)
332 pipeline = PipelineIR.from_string(pipeline_str)
334 # Create the temp file, write and read
335 with tempfile.NamedTemporaryFile() as tf:
336 pipeline.to_file(tf.name)
337 loaded_pipeline = PipelineIR.from_file(tf.name)
338 self.assertEqual(pipeline, loaded_pipeline)
340 def testPipelineYamlLoader(self):
341 # Tests that an exception is thrown in the case a key is used multiple
342 # times in a given scope within a pipeline file
343 pipeline_str = textwrap.dedent("""
344 description: Test Pipeline
345 tasks:
346 modA: test1
347 modB: test2
348 modA: test3
349 """)
350 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str)
353class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
354 pass
357def setup_module(module):
358 lsst.utils.tests.init()
361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true
362 lsst.utils.tests.init()
363 unittest.main()