Coverage for tests/test_pipelineIR.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2# This file is part of pipe_base.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
23import os
24import tempfile
25import textwrap
26import unittest
28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR
29import lsst.utils.tests
32class ConfigIRTestCase(unittest.TestCase):
33 """A test case for ConfigIR Objects
35 ConfigIR contains a method that is not exercised by the PipelineIR task,
36 so it should be tested here
37 """
39 def setUp(self):
40 pass
42 def tearDown(self):
43 pass
45 def testMergeConfig(self):
46 # Create some configs to merge
47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"],
48 rest={"a": 1, "b": 2})
49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2})
50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2})
51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4})
52 config5 = ConfigIR(rest={"f": 5, "g": 6})
53 config6 = ConfigIR(rest={"h": 7, "i": 8})
54 config7 = ConfigIR(rest={"h": 9})
56 # Merge configs with different dataIds, this should yield two elements
57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2])
59 # Merge configs with python blocks defined, this should yield two
60 # elements
61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3])
63 # Merge configs with file defined, this should yield two elements
64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4])
66 # merge config2 into config1
67 merge_result = list(config5.maybe_merge(config6))
68 self.assertEqual(len(merge_result), 1)
69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8})
71 # Cant merge configs with shared keys
72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7])
75class PipelineIRTestCase(unittest.TestCase):
76 """A test case for PipelineIR objects
77 """
79 def setUp(self):
80 pass
82 def tearDown(self):
83 pass
85 def testPipelineIRInitChecks(self):
86 # Missing description
87 pipeline_str = """
88 tasks:
89 a: module.A
90 """
91 with self.assertRaises(ValueError):
92 PipelineIR.from_string(pipeline_str)
94 # Missing tasks
95 pipeline_str = """
96 description: Test Pipeline
97 """
98 with self.assertRaises(ValueError):
99 PipelineIR.from_string(pipeline_str)
101 # This should raise a FileNotFoundError, as there are inherits defined
102 # so the __init__ method should pass but the inherited file does not
103 # exist
104 pipeline_str = textwrap.dedent("""
105 description: Test Pipeline
106 inherits: /dummy_pipeline.yaml
107 """)
109 with self.assertRaises(FileNotFoundError):
110 PipelineIR.from_string(pipeline_str)
112 def testTaskParsing(self):
113 # Should be able to parse a task defined both ways
114 pipeline_str = textwrap.dedent("""
115 description: Test Pipeline
116 tasks:
117 modA: test.modA
118 modB:
119 class: test.modB
120 """)
122 pipeline = PipelineIR.from_string(pipeline_str)
123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"])
124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"])
126 def testInheritParsing(self):
127 # This should raise, as the two pipelines, both define the same label
128 pipeline_str = textwrap.dedent("""
129 description: Test Pipeline
130 inherits:
131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
133 """)
135 with self.assertRaises(ValueError):
136 PipelineIR.from_string(pipeline_str)
138 # This should pass, as the conflicting task is excluded
139 pipeline_str = textwrap.dedent("""
140 description: Test Pipeline
141 inherits:
142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
143 exclude: modA
144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
145 """)
146 pipeline = PipelineIR.from_string(pipeline_str)
147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
149 # This should pass, as the conflicting task is no in includes
150 pipeline_str = textwrap.dedent("""
151 description: Test Pipeline
152 inherits:
153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
154 include: modB
155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
156 """)
158 pipeline = PipelineIR.from_string(pipeline_str)
159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
161 # Test that you cant include and exclude a task
162 pipeline_str = textwrap.dedent("""
163 description: Test Pipeline
164 inherits:
165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
166 exclude: modA
167 include: modB
168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
169 """)
171 with self.assertRaises(ValueError):
172 PipelineIR.from_string(pipeline_str)
174 # Test that contracts are inherited
175 pipeline_str = textwrap.dedent("""
176 description: Test Pipeline
177 inherits:
178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
179 """)
181 pipeline = PipelineIR.from_string(pipeline_str)
182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
184 # Test that contracts are not inherited
185 pipeline_str = textwrap.dedent("""
186 description: Test Pipeline
187 inherits:
188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
189 importContracts: False
190 """)
192 pipeline = PipelineIR.from_string(pipeline_str)
193 self.assertEqual(pipeline.contracts, [])
195 # Test that configs are inherited when defining the same task again
196 # with the same label
197 pipeline_str = textwrap.dedent("""
198 description: Test Pipeline
199 inherits:
200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
201 tasks:
202 modA:
203 class: "test.moduleA"
204 config:
205 value2: 2
206 """)
207 pipeline = PipelineIR.from_string(pipeline_str)
208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2})
210 # Test that configs are not inherited when redefining the task
211 # associated with a label
212 pipeline_str = textwrap.dedent("""
213 description: Test Pipeline
214 inherits:
215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
216 tasks:
217 modA:
218 class: "test.moduleAReplace"
219 config:
220 value2: 2
221 """)
222 pipeline = PipelineIR.from_string(pipeline_str)
223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2})
225 # Test that named subsets are inherited
226 pipeline_str = textwrap.dedent("""
227 description: Test Pipeline
228 inherits:
229 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
230 """)
231 pipeline = PipelineIR.from_string(pipeline_str)
232 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"})
233 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"})
235 # Test that inheriting and redeclaring a named subset works
236 pipeline_str = textwrap.dedent("""
237 description: Test Pipeline
238 inherits:
239 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
240 tasks:
241 modE: "test.moduleE"
242 subsets:
243 modSubset:
244 - modE
245 """)
246 pipeline = PipelineIR.from_string(pipeline_str)
247 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"})
248 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"})
250 # Test that inheriting from two pipelines that both declare a named
251 # subset with the same name fails
252 pipeline_str = textwrap.dedent("""
253 description: Test Pipeline
254 inherits:
255 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
256 - $PIPE_BASE_DIR/tests/testPipeline3.yaml
257 """)
258 with self.assertRaises(ValueError):
259 PipelineIR.from_string(pipeline_str)
261 # Test that inheriting a named subset that duplicates a label declared
262 # in this pipeline fails
263 pipeline_str = textwrap.dedent("""
264 description: Test Pipeline
265 inherits:
266 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
267 tasks:
268 modSubset: "test.moduleE"
269 """)
270 with self.assertRaises(ValueError):
271 PipelineIR.from_string(pipeline_str)
273 # Test that inheriting fails if a named subset and task label conflict
274 pipeline_str = textwrap.dedent("""
275 description: Test Pipeline
276 inherits:
277 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
278 - $PIPE_BASE_DIR/tests/testPipeline4.yaml
279 """)
280 with self.assertRaises(ValueError):
281 PipelineIR.from_string(pipeline_str)
283 def testReadContracts(self):
284 # Verify that contracts are read in from a pipeline
285 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml")
286 pipeline = PipelineIR.from_file(location)
287 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
289 # Verify that a contract message is loaded
290 pipeline_str = textwrap.dedent("""
291 description: Test Pipeline
292 tasks:
293 modA: test.modA
294 modB:
295 class: test.modB
296 contracts:
297 - contract: modA.foo == modB.Bar
298 msg: "Test message"
299 """)
301 pipeline = PipelineIR.from_string(pipeline_str)
302 self.assertEqual(pipeline.contracts[0].msg, "Test message")
304 def testReadNamedSubsets(self):
305 pipeline_str = textwrap.dedent("""
306 description: Test Pipeline
307 tasks:
308 modA: test.modA
309 modB:
310 class: test.modB
311 modC: test.modC
312 modD: test.modD
313 subsets:
314 subset1:
315 - modA
316 - modB
317 subset2:
318 subset:
319 - modC
320 - modD
321 description: "A test named subset"
322 """)
323 pipeline = PipelineIR.from_string(pipeline_str)
324 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"})
326 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"})
327 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None)
329 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"})
330 self.assertEqual(pipeline.labeled_subsets["subset2"].description,
331 "A test named subset")
333 # verify that forgetting a subset key is an error
334 pipeline_str = textwrap.dedent("""
335 description: Test Pipeline
336 tasks:
337 modA: test.modA
338 modB:
339 class: test.modB
340 modC: test.modC
341 modD: test.modD
342 subsets:
343 subset2:
344 sub:
345 - modC
346 - modD
347 description: "A test named subset"
348 """)
349 with self.assertRaises(ValueError):
350 PipelineIR.from_string(pipeline_str)
352 # verify putting a label in a named subset that is not in the task is
353 # an error
354 pipeline_str = textwrap.dedent("""
355 description: Test Pipeline
356 tasks:
357 modA: test.modA
358 modB:
359 class: test.modB
360 modC: test.modC
361 modD: test.modD
362 subsets:
363 subset2:
364 - modC
365 - modD
366 - modE
367 """)
368 with self.assertRaises(ValueError):
369 PipelineIR.from_string(pipeline_str)
371 def testInstrument(self):
372 # Verify that if instrument is defined it is parsed out
373 pipeline_str = textwrap.dedent("""
374 description: Test Pipeline
375 instrument: dummyCam
376 tasks:
377 modA: test.moduleA
378 """)
380 pipeline = PipelineIR.from_string(pipeline_str)
381 self.assertEqual(pipeline.instrument, "dummyCam")
383 def testReadTaskConfig(self):
384 # Verify that a task with a config is read in correctly
385 pipeline_str = textwrap.dedent("""
386 description: Test Pipeline
387 tasks:
388 modA:
389 class: test.moduleA
390 config:
391 propertyA: 6
392 propertyB: 7
393 file: testfile.py
394 python: "config.testDict['a'] = 9"
395 """)
397 pipeline = PipelineIR.from_string(pipeline_str)
398 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"])
399 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9")
400 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
402 # Verify that multiple files are read fine
403 pipeline_str = textwrap.dedent("""
404 description: Test Pipeline
405 tasks:
406 modA:
407 class: test.moduleA
408 config:
409 file:
410 - testfile.py
411 - otherFile.py
412 """)
414 pipeline = PipelineIR.from_string(pipeline_str)
415 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"])
417 # Test reading multiple Config entries
418 pipeline_str = textwrap.dedent("""
419 description: Test Pipeline
420 tasks:
421 modA:
422 class: test.moduleA
423 config:
424 - propertyA: 6
425 propertyB: 7
426 dataId: {"visit": 6}
427 - propertyA: 8
428 propertyB: 9
429 """)
431 pipeline = PipelineIR.from_string(pipeline_str)
432 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
433 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6})
434 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9})
435 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None)
437 def testSerialization(self):
438 # Test creating a pipeline, writing it to a file, reading the file
439 pipeline_str = textwrap.dedent("""
440 description: Test Pipeline
441 instrument: dummyCam
442 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml
443 tasks:
444 modC:
445 class: test.moduleC
446 config:
447 - propertyA: 6
448 propertyB: 7
449 dataId: {"visit": 6}
450 - propertyA: 8
451 propertyB: 9
452 modD: test.moduleD
453 contracts:
454 - modA.foo == modB.bar
455 subsets:
456 subA:
457 - modA
458 - modC
459 """)
461 pipeline = PipelineIR.from_string(pipeline_str)
463 # Create the temp file, write and read
464 with tempfile.NamedTemporaryFile() as tf:
465 pipeline.to_file(tf.name)
466 loaded_pipeline = PipelineIR.from_file(tf.name)
467 self.assertEqual(pipeline, loaded_pipeline)
469 def testPipelineYamlLoader(self):
470 # Tests that an exception is thrown in the case a key is used multiple
471 # times in a given scope within a pipeline file
472 pipeline_str = textwrap.dedent("""
473 description: Test Pipeline
474 tasks:
475 modA: test1
476 modB: test2
477 modA: test3
478 """)
479 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str)
482class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
483 pass
486def setup_module(module):
487 lsst.utils.tests.init()
490if __name__ == "__main__": 490 ↛ 491line 490 didn't jump to line 491, because the condition on line 490 was never true
491 lsst.utils.tests.init()
492 unittest.main()