Coverage for tests/test_pipelineIR.py: 14%

203 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-14 10:49 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import os 

29import tempfile 

30import textwrap 

31import unittest 

32 

33import lsst.utils.tests 

34from lsst.pipe.base.pipelineIR import ConfigIR, PipelineIR, PipelineSubsetCtrl 

35 

36# Find where the test pipelines exist and store it in an environment variable. 

37os.environ["TESTDIR"] = os.path.dirname(__file__) 

38 

39 

40class ConfigIRTestCase(unittest.TestCase): 

41 """A test case for ConfigIR Objects. 

42 

43 ConfigIR contains a method that is not exercised by the PipelineIR task, 

44 so it should be tested here. 

45 """ 

46 

47 def setUp(self): 

48 pass 

49 

50 def tearDown(self): 

51 pass 

52 

53 def testMergeConfig(self): 

54 # Create some configs to merge 

55 config1 = ConfigIR( 

56 python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], rest={"a": 1, "b": 2} 

57 ) 

58 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

59 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

60 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

61 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

62 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

63 config7 = ConfigIR(rest={"h": 9}) 

64 

65 # Merge configs with different dataIds, this should yield two elements 

66 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

67 

68 # Merge configs with python blocks defined, this should yield two 

69 # elements 

70 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

71 

72 # Merge configs with file defined, this should yield two elements 

73 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

74 

75 # merge config2 into config1 

76 merge_result = list(config5.maybe_merge(config6)) 

77 self.assertEqual(len(merge_result), 1) 

78 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

79 

80 # Cant merge configs with shared keys 

81 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

82 

83 

84class PipelineIRTestCase(unittest.TestCase): 

85 """A test case for PipelineIR objects.""" 

86 

87 def setUp(self): 

88 pass 

89 

90 def tearDown(self): 

91 pass 

92 

93 def testPipelineIRInitChecks(self): 

94 # Missing description 

95 pipeline_str = """ 

96 tasks: 

97 a: module.A 

98 """ 

99 with self.assertRaises(ValueError): 

100 PipelineIR.from_string(pipeline_str) 

101 

102 # Missing tasks 

103 pipeline_str = """ 

104 description: Test Pipeline 

105 """ 

106 with self.assertRaises(ValueError): 

107 PipelineIR.from_string(pipeline_str) 

108 

109 # This should raise a FileNotFoundError, as there are imported defined 

110 # so the __init__ method should pass but the imported file does not 

111 # exist 

112 pipeline_str = textwrap.dedent( 

113 """ 

114 description: Test Pipeline 

115 imports: /dummy_pipeline.yaml 

116 """ 

117 ) 

118 

119 with self.assertRaises(FileNotFoundError): 

120 PipelineIR.from_string(pipeline_str) 

121 

122 def testTaskParsing(self): 

123 # Should be able to parse a task defined both ways 

124 pipeline_str = textwrap.dedent( 

125 """ 

126 description: Test Pipeline 

127 tasks: 

128 modA: test.modA 

129 modB: 

130 class: test.modB 

131 """ 

132 ) 

133 

134 pipeline = PipelineIR.from_string(pipeline_str) 

135 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

136 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

137 

138 def testImportParsing(self): 

139 # This should raise, as the two pipelines, both define the same label 

140 pipeline_str = textwrap.dedent( 

141 """ 

142 description: Test Pipeline 

143 imports: 

144 - $TESTDIR/testPipeline1.yaml 

145 - $TESTDIR/testPipeline2.yaml 

146 """ 

147 ) 

148 # "modA" is the duplicated label, and it should appear in the error. 

149 with self.assertRaisesRegex(ValueError, "modA"): 

150 PipelineIR.from_string(pipeline_str) 

151 

152 # This should pass, as the conflicting task is excluded 

153 pipeline_str = textwrap.dedent( 

154 """ 

155 description: Test Pipeline 

156 imports: 

157 - location: $TESTDIR/testPipeline1.yaml 

158 exclude: modA 

159 - $TESTDIR/testPipeline2.yaml 

160 """ 

161 ) 

162 pipeline = PipelineIR.from_string(pipeline_str) 

163 self.assertEqual(set(pipeline.tasks.keys()), {"modA", "modB"}) 

164 

165 # This should pass, as the conflicting task is no in includes 

166 pipeline_str = textwrap.dedent( 

167 """ 

168 description: Test Pipeline 

169 imports: 

170 - location: $TESTDIR/testPipeline1.yaml 

171 include: modB 

172 labeledSubsetModifyMode: DROP 

173 - $TESTDIR/testPipeline2.yaml 

174 """ 

175 ) 

176 

177 pipeline = PipelineIR.from_string(pipeline_str) 

178 self.assertEqual(set(pipeline.tasks.keys()), {"modA", "modB"}) 

179 

180 # Test that you cant include and exclude a task 

181 pipeline_str = textwrap.dedent( 

182 """ 

183 description: Test Pipeline 

184 imports: 

185 - location: $TESTDIR/testPipeline1.yaml 

186 exclude: modA 

187 include: modB 

188 labeledSubsetModifyMode: EDIT 

189 - $TESTDIR/testPipeline2.yaml 

190 """ 

191 ) 

192 

193 with self.assertRaises(ValueError): 

194 PipelineIR.from_string(pipeline_str) 

195 

196 # Test unknown labeledSubsetModifyModes raise 

197 pipeline_str = textwrap.dedent( 

198 """ 

199 description: Test Pipeline 

200 imports: 

201 - location: $TESTDIR/testPipeline1.yaml 

202 exclude: modA 

203 include: modB 

204 labeledSubsetModifyMode: WRONG 

205 - $TESTDIR/testPipeline2.yaml 

206 """ 

207 ) 

208 with self.assertRaises(ValueError): 

209 PipelineIR.from_string(pipeline_str) 

210 

211 # Test that contracts are imported 

212 pipeline_str = textwrap.dedent( 

213 """ 

214 description: Test Pipeline 

215 imports: 

216 - $TESTDIR/testPipeline1.yaml 

217 """ 

218 ) 

219 

220 pipeline = PipelineIR.from_string(pipeline_str) 

221 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

222 

223 # Test that contracts are not imported 

224 pipeline_str = textwrap.dedent( 

225 """ 

226 description: Test Pipeline 

227 imports: 

228 - location: $TESTDIR/testPipeline1.yaml 

229 importContracts: False 

230 """ 

231 ) 

232 

233 pipeline = PipelineIR.from_string(pipeline_str) 

234 self.assertEqual(pipeline.contracts, []) 

235 

236 # Test that configs are imported when defining the same task again 

237 # with the same label 

238 pipeline_str = textwrap.dedent( 

239 """ 

240 description: Test Pipeline 

241 imports: 

242 - $TESTDIR/testPipeline2.yaml 

243 tasks: 

244 modA: 

245 class: "test.moduleA" 

246 config: 

247 value2: 2 

248 """ 

249 ) 

250 pipeline = PipelineIR.from_string(pipeline_str) 

251 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

252 

253 # Test that configs are not imported when redefining the task 

254 # associated with a label 

255 pipeline_str = textwrap.dedent( 

256 """ 

257 description: Test Pipeline 

258 imports: 

259 - $TESTDIR/testPipeline2.yaml 

260 tasks: 

261 modA: 

262 class: "test.moduleAReplace" 

263 config: 

264 value2: 2 

265 """ 

266 ) 

267 pipeline = PipelineIR.from_string(pipeline_str) 

268 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

269 

270 # Test that named subsets are imported 

271 pipeline_str = textwrap.dedent( 

272 """ 

273 description: Test Pipeline 

274 imports: 

275 - $TESTDIR/testPipeline2.yaml 

276 """ 

277 ) 

278 pipeline = PipelineIR.from_string(pipeline_str) 

279 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

280 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"}) 

281 

282 # Test that imported and redeclaring a named subset works 

283 pipeline_str = textwrap.dedent( 

284 """ 

285 description: Test Pipeline 

286 imports: 

287 - $TESTDIR/testPipeline2.yaml 

288 tasks: 

289 modE: "test.moduleE" 

290 subsets: 

291 modSubset: 

292 - modE 

293 """ 

294 ) 

295 pipeline = PipelineIR.from_string(pipeline_str) 

296 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

297 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"}) 

298 

299 # Test that imported from two pipelines that both declare a named 

300 # subset with the same name fails 

301 pipeline_str = textwrap.dedent( 

302 """ 

303 description: Test Pipeline 

304 imports: 

305 - $TESTDIR/testPipeline2.yaml 

306 - $TESTDIR/testPipeline3.yaml 

307 """ 

308 ) 

309 with self.assertRaises(ValueError): 

310 PipelineIR.from_string(pipeline_str) 

311 

312 # Test that imported a named subset that duplicates a label declared 

313 # in this pipeline fails 

314 pipeline_str = textwrap.dedent( 

315 """ 

316 description: Test Pipeline 

317 imports: 

318 - $TESTDIR/testPipeline2.yaml 

319 tasks: 

320 modSubset: "test.moduleE" 

321 """ 

322 ) 

323 with self.assertRaises(ValueError): 

324 PipelineIR.from_string(pipeline_str) 

325 

326 # Test that imported fails if a named subset and task label conflict 

327 pipeline_str = textwrap.dedent( 

328 """ 

329 description: Test Pipeline 

330 imports: 

331 - $TESTDIR/testPipeline2.yaml 

332 - $TESTDIR/testPipeline4.yaml 

333 """ 

334 ) 

335 with self.assertRaises(ValueError): 

336 PipelineIR.from_string(pipeline_str) 

337 

338 # Test that importing Pipelines with different step definitions fails 

339 pipeline_str = textwrap.dedent( 

340 """ 

341 description: Test Pipeline 

342 imports: 

343 - $TESTDIR/testPipeline5.yaml 

344 steps: 

345 - label: sub1 

346 sharding_dimensions: ['a', 'e'] 

347 """ 

348 ) 

349 with self.assertRaises(ValueError): 

350 PipelineIR.from_string(pipeline_str) 

351 

352 # Test that it does not fail if steps are excluded 

353 pipeline_str = textwrap.dedent( 

354 """ 

355 description: Test Pipeline 

356 imports: 

357 - location: $TESTDIR/testPipeline5.yaml 

358 importSteps: false 

359 steps: 

360 - label: sub1 

361 sharding_dimensions: ['a', 'e'] 

362 """ 

363 ) 

364 PipelineIR.from_string(pipeline_str) 

365 

366 # Test that importing does work 

367 pipeline_str = textwrap.dedent( 

368 """ 

369 description: Test Pipeline 

370 imports: 

371 - location: $TESTDIR/testPipeline5.yaml 

372 """ 

373 ) 

374 pipeline = PipelineIR.from_string(pipeline_str) 

375 self.assertEqual(set(step.label for step in pipeline.steps), {"sub1", "sub2"}) 

376 

377 def testSteps(self): 

378 # Test that steps definitions are created 

379 pipeline_str = textwrap.dedent( 

380 """ 

381 description: Test Pipeline 

382 tasks: 

383 modA: "test.moduleA" 

384 modB: "test.moduleB" 

385 subsets: 

386 sub1: 

387 subset: 

388 - modA 

389 - modB 

390 sub2: 

391 subset: 

392 - modA 

393 steps: 

394 - label: sub1 

395 sharding_dimensions: ['a', 'b'] 

396 - label: sub2 

397 sharding_dimensions: ['a', 'b'] 

398 """ 

399 ) 

400 pipeline = PipelineIR.from_string(pipeline_str) 

401 self.assertEqual(set(step.label for step in pipeline.steps), {"sub1", "sub2"}) 

402 

403 # Test that steps definitions must be unique 

404 pipeline_str = textwrap.dedent( 

405 """ 

406 description: Test Pipeline 

407 tasks: 

408 modA: "test.moduleA" 

409 modB: "test.moduleB" 

410 subsets: 

411 sub1: 

412 subset: 

413 - modA 

414 - modB 

415 sub2: 

416 subset: 

417 - modA 

418 steps: 

419 - label: sub1 

420 sharding_dimensions: ['a', 'b'] 

421 - label: sub1 

422 sharding_dimensions: ['a', 'b'] 

423 """ 

424 ) 

425 with self.assertRaises(ValueError): 

426 pipeline = PipelineIR.from_string(pipeline_str) 

427 

428 def testReadParameters(self): 

429 # verify that parameters section are read in from a pipeline 

430 pipeline_str = textwrap.dedent( 

431 """ 

432 description: Test Pipeline 

433 parameters: 

434 value1: A 

435 value2: B 

436 tasks: 

437 modA: ModuleA 

438 """ 

439 ) 

440 pipeline = PipelineIR.from_string(pipeline_str) 

441 self.assertEqual(pipeline.parameters.mapping, {"value1": "A", "value2": "B"}) 

442 

443 def testTaskParameterLabel(self): 

444 # verify that "parameters" cannot be used as a task label 

445 pipeline_str = textwrap.dedent( 

446 """ 

447 description: Test Pipeline 

448 tasks: 

449 parameters: modA 

450 """ 

451 ) 

452 with self.assertRaises(ValueError): 

453 PipelineIR.from_string(pipeline_str) 

454 

455 def testParameterImporting(self): 

456 # verify that importing parameters happens correctly 

457 pipeline_str = textwrap.dedent( 

458 """ 

459 description: Test Pipeline 

460 imports: 

461 - $TESTDIR/testPipeline1.yaml 

462 - location: $TESTDIR/testPipeline2.yaml 

463 exclude: 

464 - modA 

465 

466 parameters: 

467 value4: valued 

468 """ 

469 ) 

470 pipeline = PipelineIR.from_string(pipeline_str) 

471 self.assertEqual( 

472 pipeline.parameters.mapping, 

473 {"value4": "valued", "value1": "valueNew", "value2": "valueB", "value3": "valueC"}, 

474 ) 

475 

476 def testImportingInstrument(self): 

477 # verify an instrument is imported, or ignored, (Or otherwise modified 

478 # for potential future use) 

479 pipeline_str = textwrap.dedent( 

480 """ 

481 description: Test Pipeline 

482 imports: 

483 - $TESTDIR/testPipeline1.yaml 

484 """ 

485 ) 

486 pipeline = PipelineIR.from_string(pipeline_str) 

487 self.assertEqual(pipeline.instrument, "test.instrument") 

488 

489 # verify that an imported pipeline can have its instrument set to None 

490 pipeline_str = textwrap.dedent( 

491 """ 

492 description: Test Pipeline 

493 imports: 

494 - location: $TESTDIR/testPipeline1.yaml 

495 instrument: None 

496 """ 

497 ) 

498 pipeline = PipelineIR.from_string(pipeline_str) 

499 self.assertEqual(pipeline.instrument, None) 

500 

501 # verify that an imported pipeline can have its instrument modified 

502 pipeline_str = textwrap.dedent( 

503 """ 

504 description: Test Pipeline 

505 imports: 

506 - location: $TESTDIR/testPipeline1.yaml 

507 instrument: new.instrument 

508 """ 

509 ) 

510 pipeline = PipelineIR.from_string(pipeline_str) 

511 self.assertEqual(pipeline.instrument, "new.instrument") 

512 

513 # Test that multiple instruments can't be defined, 

514 # and that the error message tells you what instruments were found. 

515 pipeline_str = textwrap.dedent( 

516 """ 

517 description: Test Pipeline 

518 instrument: new.instrument 

519 imports: 

520 - location: $TESTDIR/testPipeline1.yaml 

521 """ 

522 ) 

523 with self.assertRaisesRegex(ValueError, "new.instrument .* test.instrument."): 

524 PipelineIR.from_string(pipeline_str) 

525 

526 def testParameterConfigFormatting(self): 

527 # verify that a config properly is formatted with parameters 

528 pipeline_str = textwrap.dedent( 

529 """ 

530 description: Test Pipeline 

531 parameters: 

532 value1: A 

533 tasks: 

534 modA: 

535 class: ModuleA 

536 config: 

537 testKey: parameters.value1 

538 """ 

539 ) 

540 pipeline = PipelineIR.from_string(pipeline_str) 

541 newConfig = pipeline.tasks["modA"].config[0].formatted(pipeline.parameters) 

542 self.assertEqual(newConfig.rest["testKey"], "A") 

543 

544 def testReadContracts(self): 

545 # Verify that contracts are read in from a pipeline 

546 location = "$TESTDIR/testPipeline1.yaml" 

547 pipeline = PipelineIR.from_uri(location) 

548 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

549 

550 # Verify that a contract message is loaded 

551 pipeline_str = textwrap.dedent( 

552 """ 

553 description: Test Pipeline 

554 tasks: 

555 modA: test.modA 

556 modB: 

557 class: test.modB 

558 contracts: 

559 - contract: modA.foo == modB.Bar 

560 msg: "Test message" 

561 """ 

562 ) 

563 

564 pipeline = PipelineIR.from_string(pipeline_str) 

565 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

566 

567 def testReadNamedSubsets(self): 

568 pipeline_str = textwrap.dedent( 

569 """ 

570 description: Test Pipeline 

571 tasks: 

572 modA: test.modA 

573 modB: 

574 class: test.modB 

575 modC: test.modC 

576 modD: test.modD 

577 subsets: 

578 subset1: 

579 - modA 

580 - modB 

581 subset2: 

582 subset: 

583 - modC 

584 - modD 

585 description: "A test named subset" 

586 """ 

587 ) 

588 pipeline = PipelineIR.from_string(pipeline_str) 

589 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"}) 

590 

591 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"}) 

592 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None) 

593 

594 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"}) 

595 self.assertEqual(pipeline.labeled_subsets["subset2"].description, "A test named subset") 

596 

597 # verify that forgetting a subset key is an error 

598 pipeline_str = textwrap.dedent( 

599 """ 

600 description: Test Pipeline 

601 tasks: 

602 modA: test.modA 

603 modB: 

604 class: test.modB 

605 modC: test.modC 

606 modD: test.modD 

607 subsets: 

608 subset2: 

609 sub: 

610 - modC 

611 - modD 

612 description: "A test named subset" 

613 """ 

614 ) 

615 with self.assertRaises(ValueError): 

616 PipelineIR.from_string(pipeline_str) 

617 

618 # verify putting a label in a named subset that is not in the task is 

619 # an error 

620 pipeline_str = textwrap.dedent( 

621 """ 

622 description: Test Pipeline 

623 tasks: 

624 modA: test.modA 

625 modB: 

626 class: test.modB 

627 modC: test.modC 

628 modD: test.modD 

629 subsets: 

630 subset2: 

631 - modC 

632 - modD 

633 - modE 

634 """ 

635 ) 

636 with self.assertRaises(ValueError): 

637 PipelineIR.from_string(pipeline_str) 

638 

639 def testSubsettingPipeline(self): 

640 pipeline_str = textwrap.dedent( 

641 """ 

642 description: Test Pipeline 

643 tasks: 

644 modA: test.modA 

645 modB: 

646 class: test.modB 

647 modC: test.modC 

648 modD: test.modD 

649 subsets: 

650 subset1: 

651 - modA 

652 - modB 

653 subset2: 

654 subset: 

655 - modC 

656 - modD 

657 description: "A test named subset" 

658 """ 

659 ) 

660 pipeline = PipelineIR.from_string(pipeline_str) 

661 # verify that creating a pipeline subset with the default drop behavior 

662 # removes any labeled subset that contains a label not in the set of 

663 # all task labels. 

664 pipelineSubset1 = pipeline.subset_from_labels({"modA", "modB", "modC"}) 

665 self.assertEqual(pipelineSubset1.labeled_subsets.keys(), {"subset1"}) 

666 # verify that creating a pipeline subset with the edit behavior 

667 # edits any labeled subset that contains a label not in the set of 

668 # all task labels. 

669 pipelineSubset2 = pipeline.subset_from_labels({"modA", "modB", "modC"}, PipelineSubsetCtrl.EDIT) 

670 self.assertEqual(pipelineSubset2.labeled_subsets.keys(), {"subset1", "subset2"}) 

671 self.assertEqual(pipelineSubset2.labeled_subsets["subset2"].subset, {"modC"}) 

672 

673 def testInstrument(self): 

674 # Verify that if instrument is defined it is parsed out 

675 pipeline_str = textwrap.dedent( 

676 """ 

677 description: Test Pipeline 

678 instrument: dummyCam 

679 tasks: 

680 modA: test.moduleA 

681 """ 

682 ) 

683 

684 pipeline = PipelineIR.from_string(pipeline_str) 

685 self.assertEqual(pipeline.instrument, "dummyCam") 

686 

687 def testReadTaskConfig(self): 

688 # Verify that a task with a config is read in correctly 

689 pipeline_str = textwrap.dedent( 

690 """ 

691 description: Test Pipeline 

692 tasks: 

693 modA: 

694 class: test.moduleA 

695 config: 

696 propertyA: 6 

697 propertyB: 7 

698 file: testfile.py 

699 python: "config.testDict['a'] = 9" 

700 """ 

701 ) 

702 

703 pipeline = PipelineIR.from_string(pipeline_str) 

704 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

705 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

706 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

707 

708 # Verify that multiple files are read fine 

709 pipeline_str = textwrap.dedent( 

710 """ 

711 description: Test Pipeline 

712 tasks: 

713 modA: 

714 class: test.moduleA 

715 config: 

716 file: 

717 - testfile.py 

718 - otherFile.py 

719 """ 

720 ) 

721 

722 pipeline = PipelineIR.from_string(pipeline_str) 

723 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

724 

725 # Test reading multiple Config entries 

726 pipeline_str = textwrap.dedent( 

727 """ 

728 description: Test Pipeline 

729 tasks: 

730 modA: 

731 class: test.moduleA 

732 config: 

733 - propertyA: 6 

734 propertyB: 7 

735 dataId: {"visit": 6} 

736 - propertyA: 8 

737 propertyB: 9 

738 """ 

739 ) 

740 

741 pipeline = PipelineIR.from_string(pipeline_str) 

742 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

743 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

744 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

745 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

746 

747 def testSerialization(self): 

748 # Test creating a pipeline, writing it to a file, reading the file 

749 pipeline_str = textwrap.dedent( 

750 """ 

751 description: Test Pipeline 

752 instrument: dummyCam 

753 imports: 

754 - location: $TESTDIR/testPipeline1.yaml 

755 instrument: None 

756 tasks: 

757 modC: 

758 class: test.moduleC 

759 config: 

760 - propertyA: 6 

761 propertyB: 7 

762 dataId: {"visit": 6} 

763 - propertyA: 8 

764 propertyB: 9 

765 modD: test.moduleD 

766 contracts: 

767 - modA.foo == modB.bar 

768 subsets: 

769 subA: 

770 - modA 

771 - modC 

772 """ 

773 ) 

774 

775 pipeline = PipelineIR.from_string(pipeline_str) 

776 

777 # Create the temp file, write and read 

778 with tempfile.NamedTemporaryFile() as tf: 

779 pipeline.write_to_uri(tf.name) 

780 loaded_pipeline = PipelineIR.from_uri(tf.name) 

781 self.assertEqual(pipeline, loaded_pipeline) 

782 

783 def testPipelineYamlLoader(self): 

784 # Tests that an exception is thrown in the case a key is used multiple 

785 # times in a given scope within a pipeline file 

786 pipeline_str = textwrap.dedent( 

787 """ 

788 description: Test Pipeline 

789 tasks: 

790 modA: test1 

791 modB: test2 

792 modA: test3 

793 """ 

794 ) 

795 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

796 

797 def testMultiLineStrings(self): 

798 """Test that multi-line strings in pipelines are written with 

799 '|' continuation-syntax instead of explicit newlines. 

800 """ 

801 pipeline_ir = PipelineIR({"description": "Line 1\nLine2\n", "tasks": {"modA": "task1"}}) 

802 string = str(pipeline_ir) 

803 self.assertIn("|", string) 

804 self.assertNotIn(r"\n", string) 

805 

806 

807class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

808 """Run file leak tests.""" 

809 

810 

811def setup_module(module): 

812 """Configure pytest.""" 

813 lsst.utils.tests.init() 

814 

815 

816if __name__ == "__main__": 

817 lsst.utils.tests.init() 

818 unittest.main()