Coverage for tests/test_pipelineIR.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2# This file is part of pipe_base.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
23import os
24import tempfile
25import textwrap
26import unittest
28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR
29import lsst.utils.tests
32class ConfigIRTestCase(unittest.TestCase):
33 """A test case for ConfigIR Objects
35 ConfigIR contains a method that is not exercised by the PipelineIR task,
36 so it should be tested here
37 """
39 def setUp(self):
40 pass
42 def tearDown(self):
43 pass
45 def testMergeConfig(self):
46 # Create some configs to merge
47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"],
48 rest={"a": 1, "b": 2})
49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2})
50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2})
51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4})
52 config5 = ConfigIR(rest={"f": 5, "g": 6})
53 config6 = ConfigIR(rest={"h": 7, "i": 8})
54 config7 = ConfigIR(rest={"h": 9})
56 # Merge configs with different dataIds, this should yield two elements
57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2])
59 # Merge configs with python blocks defined, this should yield two
60 # elements
61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3])
63 # Merge configs with file defined, this should yield two elements
64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4])
66 # merge config2 into config1
67 merge_result = list(config5.maybe_merge(config6))
68 self.assertEqual(len(merge_result), 1)
69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8})
71 # Cant merge configs with shared keys
72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7])
75class PipelineIRTestCase(unittest.TestCase):
76 """A test case for PipelineIR objects
77 """
79 def setUp(self):
80 pass
82 def tearDown(self):
83 pass
85 def testPipelineIRInitChecks(self):
86 # Missing description
87 pipeline_str = """
88 tasks:
89 a: module.A
90 """
91 with self.assertRaises(ValueError):
92 PipelineIR.from_string(pipeline_str)
94 # Missing tasks
95 pipeline_str = """
96 description: Test Pipeline
97 """
98 with self.assertRaises(ValueError):
99 PipelineIR.from_string(pipeline_str)
101 # This should raise a FileNotFoundError, as there are inherits defined
102 # so the __init__ method should pass but the inherited file does not
103 # exist
104 pipeline_str = textwrap.dedent("""
105 description: Test Pipeline
106 inherits: /dummy_pipeline.yaml
107 """)
109 with self.assertRaises(FileNotFoundError):
110 PipelineIR.from_string(pipeline_str)
112 def testTaskParsing(self):
113 # Should be able to parse a task defined both ways
114 pipeline_str = textwrap.dedent("""
115 description: Test Pipeline
116 tasks:
117 modA: test.modA
118 modB:
119 class: test.modB
120 """)
122 pipeline = PipelineIR.from_string(pipeline_str)
123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"])
124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"])
126 def testInheritParsing(self):
127 # This should raise, as the two pipelines, both define the same label
128 pipeline_str = textwrap.dedent("""
129 description: Test Pipeline
130 inherits:
131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
133 """)
135 with self.assertRaises(ValueError):
136 PipelineIR.from_string(pipeline_str)
138 # This should pass, as the conflicting task is excluded
139 pipeline_str = textwrap.dedent("""
140 description: Test Pipeline
141 inherits:
142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
143 exclude: modA
144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
145 """)
146 pipeline = PipelineIR.from_string(pipeline_str)
147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
149 # This should pass, as the conflicting task is no in includes
150 pipeline_str = textwrap.dedent("""
151 description: Test Pipeline
152 inherits:
153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
154 include: modB
155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
156 """)
158 pipeline = PipelineIR.from_string(pipeline_str)
159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"]))
161 # Test that you cant include and exclude a task
162 pipeline_str = textwrap.dedent("""
163 description: Test Pipeline
164 inherits:
165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
166 exclude: modA
167 include: modB
168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
169 """)
171 with self.assertRaises(ValueError):
172 PipelineIR.from_string(pipeline_str)
174 # Test that contracts are inherited
175 pipeline_str = textwrap.dedent("""
176 description: Test Pipeline
177 inherits:
178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
179 """)
181 pipeline = PipelineIR.from_string(pipeline_str)
182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
184 # Test that contracts are not inherited
185 pipeline_str = textwrap.dedent("""
186 description: Test Pipeline
187 inherits:
188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml
189 importContracts: False
190 """)
192 pipeline = PipelineIR.from_string(pipeline_str)
193 self.assertEqual(pipeline.contracts, [])
195 # Test that configs are inherited when defining the same task again
196 # with the same label
197 pipeline_str = textwrap.dedent("""
198 description: Test Pipeline
199 inherits:
200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
201 tasks:
202 modA:
203 class: "test.moduleA"
204 config:
205 value2: 2
206 """)
207 pipeline = PipelineIR.from_string(pipeline_str)
208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2})
210 # Test that configs are not inherited when redefining the task
211 # associated with a label
212 pipeline_str = textwrap.dedent("""
213 description: Test Pipeline
214 inherits:
215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
216 tasks:
217 modA:
218 class: "test.moduleAReplace"
219 config:
220 value2: 2
221 """)
222 pipeline = PipelineIR.from_string(pipeline_str)
223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2})
225 # Test that named subsets are inherited
226 pipeline_str = textwrap.dedent("""
227 description: Test Pipeline
228 inherits:
229 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
230 """)
231 pipeline = PipelineIR.from_string(pipeline_str)
232 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"})
233 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"})
235 # Test that inheriting and redeclaring a named subset works
236 pipeline_str = textwrap.dedent("""
237 description: Test Pipeline
238 inherits:
239 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
240 tasks:
241 modE: "test.moduleE"
242 subsets:
243 modSubset:
244 - modE
245 """)
246 pipeline = PipelineIR.from_string(pipeline_str)
247 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"})
248 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"})
250 # Test that inheriting from two pipelines that both declare a named
251 # subset with the same name fails
252 pipeline_str = textwrap.dedent("""
253 description: Test Pipeline
254 inherits:
255 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
256 - $PIPE_BASE_DIR/tests/testPipeline3.yaml
257 """)
258 with self.assertRaises(ValueError):
259 PipelineIR.from_string(pipeline_str)
261 # Test that inheriting a named subset that duplicates a label declared
262 # in this pipeline fails
263 pipeline_str = textwrap.dedent("""
264 description: Test Pipeline
265 inherits:
266 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
267 tasks:
268 modSubset: "test.moduleE"
269 """)
270 with self.assertRaises(ValueError):
271 PipelineIR.from_string(pipeline_str)
273 # Test that inheriting fails if a named subset and task label conflict
274 pipeline_str = textwrap.dedent("""
275 description: Test Pipeline
276 inherits:
277 - $PIPE_BASE_DIR/tests/testPipeline2.yaml
278 - $PIPE_BASE_DIR/tests/testPipeline4.yaml
279 """)
280 with self.assertRaises(ValueError):
281 PipelineIR.from_string(pipeline_str)
283 def testReadParameters(self):
284 # verify that parameters section are read in from a pipeline
285 pipeline_str = textwrap.dedent("""
286 description: Test Pipeline
287 parameters:
288 value1: A
289 value2: B
290 tasks:
291 modA: ModuleA
292 """)
293 pipeline = PipelineIR.from_string(pipeline_str)
294 self.assertEqual(pipeline.parameters.mapping, {"value1": "A", "value2": "B"})
296 def testTaskParameterLabel(self):
297 # verify that "parameters" cannot be used as a task label
298 pipeline_str = textwrap.dedent("""
299 description: Test Pipeline
300 tasks:
301 parameters: modA
302 """)
303 with self.assertRaises(ValueError):
304 PipelineIR.from_string(pipeline_str)
306 def testParameterInheritance(self):
307 # verify that inheriting parameters happens correctly
308 pipeline_str = textwrap.dedent("""
309 description: Test Pipeline
310 inherits:
311 - $PIPE_BASE_DIR/tests/testPipeline1.yaml
312 - location: $PIPE_BASE_DIR/tests/testPipeline2.yaml
313 exclude:
314 - modA
316 parameters:
317 value4: valued
318 """)
319 pipeline = PipelineIR.from_string(pipeline_str)
320 self.assertEqual(pipeline.parameters.mapping, {"value4": "valued", "value1": "valueNew",
321 "value2": "valueB",
322 "value3": "valueC"})
324 def testParameterConfigFormatting(self):
325 # verify that a config properly is formatted with parameters
326 pipeline_str = textwrap.dedent("""
327 description: Test Pipeline
328 parameters:
329 value1: A
330 tasks:
331 modA:
332 class: ModuleA
333 config:
334 testKey: parameters.value1
335 """)
336 pipeline = PipelineIR.from_string(pipeline_str)
337 newConfig = pipeline.tasks['modA'].config[0].formatted(pipeline.parameters)
338 self.assertEqual(newConfig.rest['testKey'], "A")
340 def testReadContracts(self):
341 # Verify that contracts are read in from a pipeline
342 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml")
343 pipeline = PipelineIR.from_file(location)
344 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c")
346 # Verify that a contract message is loaded
347 pipeline_str = textwrap.dedent("""
348 description: Test Pipeline
349 tasks:
350 modA: test.modA
351 modB:
352 class: test.modB
353 contracts:
354 - contract: modA.foo == modB.Bar
355 msg: "Test message"
356 """)
358 pipeline = PipelineIR.from_string(pipeline_str)
359 self.assertEqual(pipeline.contracts[0].msg, "Test message")
361 def testReadNamedSubsets(self):
362 pipeline_str = textwrap.dedent("""
363 description: Test Pipeline
364 tasks:
365 modA: test.modA
366 modB:
367 class: test.modB
368 modC: test.modC
369 modD: test.modD
370 subsets:
371 subset1:
372 - modA
373 - modB
374 subset2:
375 subset:
376 - modC
377 - modD
378 description: "A test named subset"
379 """)
380 pipeline = PipelineIR.from_string(pipeline_str)
381 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"})
383 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"})
384 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None)
386 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"})
387 self.assertEqual(pipeline.labeled_subsets["subset2"].description,
388 "A test named subset")
390 # verify that forgetting a subset key is an error
391 pipeline_str = textwrap.dedent("""
392 description: Test Pipeline
393 tasks:
394 modA: test.modA
395 modB:
396 class: test.modB
397 modC: test.modC
398 modD: test.modD
399 subsets:
400 subset2:
401 sub:
402 - modC
403 - modD
404 description: "A test named subset"
405 """)
406 with self.assertRaises(ValueError):
407 PipelineIR.from_string(pipeline_str)
409 # verify putting a label in a named subset that is not in the task is
410 # an error
411 pipeline_str = textwrap.dedent("""
412 description: Test Pipeline
413 tasks:
414 modA: test.modA
415 modB:
416 class: test.modB
417 modC: test.modC
418 modD: test.modD
419 subsets:
420 subset2:
421 - modC
422 - modD
423 - modE
424 """)
425 with self.assertRaises(ValueError):
426 PipelineIR.from_string(pipeline_str)
428 def testInstrument(self):
429 # Verify that if instrument is defined it is parsed out
430 pipeline_str = textwrap.dedent("""
431 description: Test Pipeline
432 instrument: dummyCam
433 tasks:
434 modA: test.moduleA
435 """)
437 pipeline = PipelineIR.from_string(pipeline_str)
438 self.assertEqual(pipeline.instrument, "dummyCam")
440 def testReadTaskConfig(self):
441 # Verify that a task with a config is read in correctly
442 pipeline_str = textwrap.dedent("""
443 description: Test Pipeline
444 tasks:
445 modA:
446 class: test.moduleA
447 config:
448 propertyA: 6
449 propertyB: 7
450 file: testfile.py
451 python: "config.testDict['a'] = 9"
452 """)
454 pipeline = PipelineIR.from_string(pipeline_str)
455 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"])
456 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9")
457 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
459 # Verify that multiple files are read fine
460 pipeline_str = textwrap.dedent("""
461 description: Test Pipeline
462 tasks:
463 modA:
464 class: test.moduleA
465 config:
466 file:
467 - testfile.py
468 - otherFile.py
469 """)
471 pipeline = PipelineIR.from_string(pipeline_str)
472 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"])
474 # Test reading multiple Config entries
475 pipeline_str = textwrap.dedent("""
476 description: Test Pipeline
477 tasks:
478 modA:
479 class: test.moduleA
480 config:
481 - propertyA: 6
482 propertyB: 7
483 dataId: {"visit": 6}
484 - propertyA: 8
485 propertyB: 9
486 """)
488 pipeline = PipelineIR.from_string(pipeline_str)
489 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7})
490 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6})
491 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9})
492 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None)
494 def testSerialization(self):
495 # Test creating a pipeline, writing it to a file, reading the file
496 pipeline_str = textwrap.dedent("""
497 description: Test Pipeline
498 instrument: dummyCam
499 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml
500 tasks:
501 modC:
502 class: test.moduleC
503 config:
504 - propertyA: 6
505 propertyB: 7
506 dataId: {"visit": 6}
507 - propertyA: 8
508 propertyB: 9
509 modD: test.moduleD
510 contracts:
511 - modA.foo == modB.bar
512 subsets:
513 subA:
514 - modA
515 - modC
516 """)
518 pipeline = PipelineIR.from_string(pipeline_str)
520 # Create the temp file, write and read
521 with tempfile.NamedTemporaryFile() as tf:
522 pipeline.to_file(tf.name)
523 loaded_pipeline = PipelineIR.from_file(tf.name)
524 self.assertEqual(pipeline, loaded_pipeline)
526 def testPipelineYamlLoader(self):
527 # Tests that an exception is thrown in the case a key is used multiple
528 # times in a given scope within a pipeline file
529 pipeline_str = textwrap.dedent("""
530 description: Test Pipeline
531 tasks:
532 modA: test1
533 modB: test2
534 modA: test3
535 """)
536 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str)
539class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
540 pass
543def setup_module(module):
544 lsst.utils.tests.init()
547if __name__ == "__main__": 547 ↛ 548line 547 didn't jump to line 548, because the condition on line 547 was never true
548 lsst.utils.tests.init()
549 unittest.main()