Coverage for tests/test_pipelineIR.py: 12%

195 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-07 02:48 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import os 

29import tempfile 

30import textwrap 

31import unittest 

32 

33import lsst.utils.tests 

34from lsst.pipe.base.pipelineIR import ConfigIR, PipelineIR, PipelineSubsetCtrl 

35 

36# Find where the test pipelines exist and store it in an environment variable. 

37os.environ["TESTDIR"] = os.path.dirname(__file__) 

38 

39 

40class ConfigIRTestCase(unittest.TestCase): 

41 """A test case for ConfigIR Objects. 

42 

43 ConfigIR contains a method that is not exercised by the PipelineIR task, 

44 so it should be tested here. 

45 """ 

46 

47 def testMergeConfig(self): 

48 # Create some configs to merge 

49 config1 = ConfigIR( 

50 python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], rest={"a": 1, "b": 2} 

51 ) 

52 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

53 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

54 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

55 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

56 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

57 config7 = ConfigIR(rest={"h": 9}) 

58 

59 # Merge configs with different dataIds, this should yield two elements 

60 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

61 

62 # Merge configs with python blocks defined, this should yield two 

63 # elements 

64 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

65 

66 # Merge configs with file defined, this should yield two elements 

67 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

68 

69 # merge config2 into config1 

70 merge_result = list(config5.maybe_merge(config6)) 

71 self.assertEqual(len(merge_result), 1) 

72 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

73 

74 # Cant merge configs with shared keys 

75 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

76 

77 

78class PipelineIRTestCase(unittest.TestCase): 

79 """A test case for PipelineIR objects.""" 

80 

81 def testPipelineIRInitChecks(self): 

82 # Missing description 

83 pipeline_str = """ 

84 tasks: 

85 a: module.A 

86 """ 

87 with self.assertRaises(ValueError): 

88 PipelineIR.from_string(pipeline_str) 

89 

90 # Missing tasks 

91 pipeline_str = """ 

92 description: Test Pipeline 

93 """ 

94 with self.assertRaises(ValueError): 

95 PipelineIR.from_string(pipeline_str) 

96 

97 # This should raise a FileNotFoundError, as there are imported defined 

98 # so the __init__ method should pass but the imported file does not 

99 # exist 

100 pipeline_str = textwrap.dedent( 

101 """ 

102 description: Test Pipeline 

103 imports: /dummy_pipeline.yaml 

104 """ 

105 ) 

106 

107 with self.assertRaises(FileNotFoundError): 

108 PipelineIR.from_string(pipeline_str) 

109 

110 def testTaskParsing(self): 

111 # Should be able to parse a task defined both ways 

112 pipeline_str = textwrap.dedent( 

113 """ 

114 description: Test Pipeline 

115 tasks: 

116 modA: test.modA 

117 modB: 

118 class: test.modB 

119 """ 

120 ) 

121 

122 pipeline = PipelineIR.from_string(pipeline_str) 

123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

125 

126 def testImportParsing(self): 

127 # This should raise, as the two pipelines, both define the same label 

128 pipeline_str = textwrap.dedent( 

129 """ 

130 description: Test Pipeline 

131 imports: 

132 - $TESTDIR/testPipeline1.yaml 

133 - $TESTDIR/testPipeline2.yaml 

134 """ 

135 ) 

136 # "modA" is the duplicated label, and it should appear in the error. 

137 with self.assertRaisesRegex(ValueError, "modA"): 

138 PipelineIR.from_string(pipeline_str) 

139 

140 # This should pass, as the conflicting task is excluded 

141 pipeline_str = textwrap.dedent( 

142 """ 

143 description: Test Pipeline 

144 imports: 

145 - location: $TESTDIR/testPipeline1.yaml 

146 exclude: modA 

147 - $TESTDIR/testPipeline2.yaml 

148 """ 

149 ) 

150 pipeline = PipelineIR.from_string(pipeline_str) 

151 self.assertEqual(set(pipeline.tasks.keys()), {"modA", "modB"}) 

152 

153 # This should pass, as the conflicting task is no in includes 

154 pipeline_str = textwrap.dedent( 

155 """ 

156 description: Test Pipeline 

157 imports: 

158 - location: $TESTDIR/testPipeline1.yaml 

159 include: modB 

160 labeledSubsetModifyMode: DROP 

161 - $TESTDIR/testPipeline2.yaml 

162 """ 

163 ) 

164 

165 pipeline = PipelineIR.from_string(pipeline_str) 

166 self.assertEqual(set(pipeline.tasks.keys()), {"modA", "modB"}) 

167 

168 # Test that you cant include and exclude a task 

169 pipeline_str = textwrap.dedent( 

170 """ 

171 description: Test Pipeline 

172 imports: 

173 - location: $TESTDIR/testPipeline1.yaml 

174 exclude: modA 

175 include: modB 

176 labeledSubsetModifyMode: EDIT 

177 - $TESTDIR/testPipeline2.yaml 

178 """ 

179 ) 

180 

181 with self.assertRaises(ValueError): 

182 PipelineIR.from_string(pipeline_str) 

183 

184 # Test unknown labeledSubsetModifyModes raise 

185 pipeline_str = textwrap.dedent( 

186 """ 

187 description: Test Pipeline 

188 imports: 

189 - location: $TESTDIR/testPipeline1.yaml 

190 exclude: modA 

191 include: modB 

192 labeledSubsetModifyMode: WRONG 

193 - $TESTDIR/testPipeline2.yaml 

194 """ 

195 ) 

196 with self.assertRaises(ValueError): 

197 PipelineIR.from_string(pipeline_str) 

198 

199 # Test that contracts are imported 

200 pipeline_str = textwrap.dedent( 

201 """ 

202 description: Test Pipeline 

203 imports: 

204 - $TESTDIR/testPipeline1.yaml 

205 """ 

206 ) 

207 

208 pipeline = PipelineIR.from_string(pipeline_str) 

209 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

210 

211 # Test that contracts are not imported 

212 pipeline_str = textwrap.dedent( 

213 """ 

214 description: Test Pipeline 

215 imports: 

216 - location: $TESTDIR/testPipeline1.yaml 

217 importContracts: False 

218 """ 

219 ) 

220 

221 pipeline = PipelineIR.from_string(pipeline_str) 

222 self.assertEqual(pipeline.contracts, []) 

223 

224 # Test that configs are imported when defining the same task again 

225 # with the same label 

226 pipeline_str = textwrap.dedent( 

227 """ 

228 description: Test Pipeline 

229 imports: 

230 - $TESTDIR/testPipeline2.yaml 

231 tasks: 

232 modA: 

233 class: "test.moduleA" 

234 config: 

235 value2: 2 

236 """ 

237 ) 

238 pipeline = PipelineIR.from_string(pipeline_str) 

239 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

240 

241 # Test that configs are not imported when redefining the task 

242 # associated with a label 

243 pipeline_str = textwrap.dedent( 

244 """ 

245 description: Test Pipeline 

246 imports: 

247 - $TESTDIR/testPipeline2.yaml 

248 tasks: 

249 modA: 

250 class: "test.moduleAReplace" 

251 config: 

252 value2: 2 

253 """ 

254 ) 

255 pipeline = PipelineIR.from_string(pipeline_str) 

256 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

257 

258 # Test that named subsets are imported 

259 pipeline_str = textwrap.dedent( 

260 """ 

261 description: Test Pipeline 

262 imports: 

263 - $TESTDIR/testPipeline2.yaml 

264 """ 

265 ) 

266 pipeline = PipelineIR.from_string(pipeline_str) 

267 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

268 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"}) 

269 

270 # Test that imported and redeclaring a named subset works 

271 pipeline_str = textwrap.dedent( 

272 """ 

273 description: Test Pipeline 

274 imports: 

275 - $TESTDIR/testPipeline2.yaml 

276 tasks: 

277 modE: "test.moduleE" 

278 subsets: 

279 modSubset: 

280 - modE 

281 """ 

282 ) 

283 pipeline = PipelineIR.from_string(pipeline_str) 

284 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

285 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"}) 

286 

287 # Test that imported from two pipelines that both declare a named 

288 # subset with the same name fails 

289 pipeline_str = textwrap.dedent( 

290 """ 

291 description: Test Pipeline 

292 imports: 

293 - $TESTDIR/testPipeline2.yaml 

294 - $TESTDIR/testPipeline3.yaml 

295 """ 

296 ) 

297 with self.assertRaises(ValueError): 

298 PipelineIR.from_string(pipeline_str) 

299 

300 # Test that imported a named subset that duplicates a label declared 

301 # in this pipeline fails 

302 pipeline_str = textwrap.dedent( 

303 """ 

304 description: Test Pipeline 

305 imports: 

306 - $TESTDIR/testPipeline2.yaml 

307 tasks: 

308 modSubset: "test.moduleE" 

309 """ 

310 ) 

311 with self.assertRaises(ValueError): 

312 PipelineIR.from_string(pipeline_str) 

313 

314 # Test that imported fails if a named subset and task label conflict 

315 pipeline_str = textwrap.dedent( 

316 """ 

317 description: Test Pipeline 

318 imports: 

319 - $TESTDIR/testPipeline2.yaml 

320 - $TESTDIR/testPipeline4.yaml 

321 """ 

322 ) 

323 with self.assertRaises(ValueError): 

324 PipelineIR.from_string(pipeline_str) 

325 

326 # Test that importing Pipelines with different step definitions fails 

327 pipeline_str = textwrap.dedent( 

328 """ 

329 description: Test Pipeline 

330 imports: 

331 - $TESTDIR/testPipeline5.yaml 

332 steps: 

333 - label: sub1 

334 sharding_dimensions: ['a', 'e'] 

335 """ 

336 ) 

337 with self.assertRaises(ValueError): 

338 PipelineIR.from_string(pipeline_str) 

339 

340 # Test that it does not fail if steps are excluded 

341 pipeline_str = textwrap.dedent( 

342 """ 

343 description: Test Pipeline 

344 imports: 

345 - location: $TESTDIR/testPipeline5.yaml 

346 importSteps: false 

347 steps: 

348 - label: sub1 

349 sharding_dimensions: ['a', 'e'] 

350 """ 

351 ) 

352 PipelineIR.from_string(pipeline_str) 

353 

354 # Test that importing does work 

355 pipeline_str = textwrap.dedent( 

356 """ 

357 description: Test Pipeline 

358 imports: 

359 - location: $TESTDIR/testPipeline5.yaml 

360 """ 

361 ) 

362 pipeline = PipelineIR.from_string(pipeline_str) 

363 self.assertEqual(set(step.label for step in pipeline.steps), {"sub1", "sub2"}) 

364 

365 def testSteps(self): 

366 # Test that steps definitions are created 

367 pipeline_str = textwrap.dedent( 

368 """ 

369 description: Test Pipeline 

370 tasks: 

371 modA: "test.moduleA" 

372 modB: "test.moduleB" 

373 subsets: 

374 sub1: 

375 subset: 

376 - modA 

377 - modB 

378 sub2: 

379 subset: 

380 - modA 

381 steps: 

382 - label: sub1 

383 sharding_dimensions: ['a', 'b'] 

384 - label: sub2 

385 sharding_dimensions: ['a', 'b'] 

386 """ 

387 ) 

388 pipeline = PipelineIR.from_string(pipeline_str) 

389 self.assertEqual(set(step.label for step in pipeline.steps), {"sub1", "sub2"}) 

390 

391 # Test that steps definitions must be unique 

392 pipeline_str = textwrap.dedent( 

393 """ 

394 description: Test Pipeline 

395 tasks: 

396 modA: "test.moduleA" 

397 modB: "test.moduleB" 

398 subsets: 

399 sub1: 

400 subset: 

401 - modA 

402 - modB 

403 sub2: 

404 subset: 

405 - modA 

406 steps: 

407 - label: sub1 

408 sharding_dimensions: ['a', 'b'] 

409 - label: sub1 

410 sharding_dimensions: ['a', 'b'] 

411 """ 

412 ) 

413 with self.assertRaises(ValueError): 

414 pipeline = PipelineIR.from_string(pipeline_str) 

415 

416 def testReadParameters(self): 

417 # verify that parameters section are read in from a pipeline 

418 pipeline_str = textwrap.dedent( 

419 """ 

420 description: Test Pipeline 

421 parameters: 

422 value1: A 

423 value2: B 

424 tasks: 

425 modA: ModuleA 

426 """ 

427 ) 

428 pipeline = PipelineIR.from_string(pipeline_str) 

429 self.assertEqual(pipeline.parameters.mapping, {"value1": "A", "value2": "B"}) 

430 

431 def testTaskParameterLabel(self): 

432 # verify that "parameters" cannot be used as a task label 

433 pipeline_str = textwrap.dedent( 

434 """ 

435 description: Test Pipeline 

436 tasks: 

437 parameters: modA 

438 """ 

439 ) 

440 with self.assertRaises(ValueError): 

441 PipelineIR.from_string(pipeline_str) 

442 

443 def testParameterImporting(self): 

444 # verify that importing parameters happens correctly 

445 pipeline_str = textwrap.dedent( 

446 """ 

447 description: Test Pipeline 

448 imports: 

449 - $TESTDIR/testPipeline1.yaml 

450 - location: $TESTDIR/testPipeline2.yaml 

451 exclude: 

452 - modA 

453 

454 parameters: 

455 value4: valued 

456 """ 

457 ) 

458 pipeline = PipelineIR.from_string(pipeline_str) 

459 self.assertEqual( 

460 pipeline.parameters.mapping, 

461 {"value4": "valued", "value1": "valueNew", "value2": "valueB", "value3": "valueC"}, 

462 ) 

463 

464 def testImportingInstrument(self): 

465 # verify an instrument is imported, or ignored, (Or otherwise modified 

466 # for potential future use) 

467 pipeline_str = textwrap.dedent( 

468 """ 

469 description: Test Pipeline 

470 imports: 

471 - $TESTDIR/testPipeline1.yaml 

472 """ 

473 ) 

474 pipeline = PipelineIR.from_string(pipeline_str) 

475 self.assertEqual(pipeline.instrument, "test.instrument") 

476 

477 # verify that an imported pipeline can have its instrument set to None 

478 pipeline_str = textwrap.dedent( 

479 """ 

480 description: Test Pipeline 

481 imports: 

482 - location: $TESTDIR/testPipeline1.yaml 

483 instrument: None 

484 """ 

485 ) 

486 pipeline = PipelineIR.from_string(pipeline_str) 

487 self.assertEqual(pipeline.instrument, None) 

488 

489 # verify that an imported pipeline can have its instrument modified 

490 pipeline_str = textwrap.dedent( 

491 """ 

492 description: Test Pipeline 

493 imports: 

494 - location: $TESTDIR/testPipeline1.yaml 

495 instrument: new.instrument 

496 """ 

497 ) 

498 pipeline = PipelineIR.from_string(pipeline_str) 

499 self.assertEqual(pipeline.instrument, "new.instrument") 

500 

501 # Test that multiple instruments can't be defined, 

502 # and that the error message tells you what instruments were found. 

503 pipeline_str = textwrap.dedent( 

504 """ 

505 description: Test Pipeline 

506 instrument: new.instrument 

507 imports: 

508 - location: $TESTDIR/testPipeline1.yaml 

509 """ 

510 ) 

511 with self.assertRaisesRegex(ValueError, "new.instrument .* test.instrument."): 

512 PipelineIR.from_string(pipeline_str) 

513 

514 def testParameterConfigFormatting(self): 

515 # verify that a config properly is formatted with parameters 

516 pipeline_str = textwrap.dedent( 

517 """ 

518 description: Test Pipeline 

519 parameters: 

520 value1: A 

521 tasks: 

522 modA: 

523 class: ModuleA 

524 config: 

525 testKey: parameters.value1 

526 """ 

527 ) 

528 pipeline = PipelineIR.from_string(pipeline_str) 

529 newConfig = pipeline.tasks["modA"].config[0].formatted(pipeline.parameters) 

530 self.assertEqual(newConfig.rest["testKey"], "A") 

531 

532 def testReadContracts(self): 

533 # Verify that contracts are read in from a pipeline 

534 location = "$TESTDIR/testPipeline1.yaml" 

535 pipeline = PipelineIR.from_uri(location) 

536 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

537 

538 # Verify that a contract message is loaded 

539 pipeline_str = textwrap.dedent( 

540 """ 

541 description: Test Pipeline 

542 tasks: 

543 modA: test.modA 

544 modB: 

545 class: test.modB 

546 contracts: 

547 - contract: modA.foo == modB.Bar 

548 msg: "Test message" 

549 """ 

550 ) 

551 

552 pipeline = PipelineIR.from_string(pipeline_str) 

553 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

554 

555 def testReadNamedSubsets(self): 

556 pipeline_str = textwrap.dedent( 

557 """ 

558 description: Test Pipeline 

559 tasks: 

560 modA: test.modA 

561 modB: 

562 class: test.modB 

563 modC: test.modC 

564 modD: test.modD 

565 subsets: 

566 subset1: 

567 - modA 

568 - modB 

569 subset2: 

570 subset: 

571 - modC 

572 - modD 

573 description: "A test named subset" 

574 """ 

575 ) 

576 pipeline = PipelineIR.from_string(pipeline_str) 

577 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"}) 

578 

579 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"}) 

580 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None) 

581 

582 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"}) 

583 self.assertEqual(pipeline.labeled_subsets["subset2"].description, "A test named subset") 

584 

585 # verify that forgetting a subset key is an error 

586 pipeline_str = textwrap.dedent( 

587 """ 

588 description: Test Pipeline 

589 tasks: 

590 modA: test.modA 

591 modB: 

592 class: test.modB 

593 modC: test.modC 

594 modD: test.modD 

595 subsets: 

596 subset2: 

597 sub: 

598 - modC 

599 - modD 

600 description: "A test named subset" 

601 """ 

602 ) 

603 with self.assertRaises(ValueError): 

604 PipelineIR.from_string(pipeline_str) 

605 

606 # verify putting a label in a named subset that is not in the task is 

607 # an error 

608 pipeline_str = textwrap.dedent( 

609 """ 

610 description: Test Pipeline 

611 tasks: 

612 modA: test.modA 

613 modB: 

614 class: test.modB 

615 modC: test.modC 

616 modD: test.modD 

617 subsets: 

618 subset2: 

619 - modC 

620 - modD 

621 - modE 

622 """ 

623 ) 

624 with self.assertRaises(ValueError): 

625 PipelineIR.from_string(pipeline_str) 

626 

627 def testSubsettingPipeline(self): 

628 pipeline_str = textwrap.dedent( 

629 """ 

630 description: Test Pipeline 

631 tasks: 

632 modA: test.modA 

633 modB: 

634 class: test.modB 

635 modC: test.modC 

636 modD: test.modD 

637 subsets: 

638 subset1: 

639 - modA 

640 - modB 

641 subset2: 

642 subset: 

643 - modC 

644 - modD 

645 description: "A test named subset" 

646 """ 

647 ) 

648 pipeline = PipelineIR.from_string(pipeline_str) 

649 # verify that creating a pipeline subset with the default drop behavior 

650 # removes any labeled subset that contains a label not in the set of 

651 # all task labels. 

652 pipelineSubset1 = pipeline.subset_from_labels({"modA", "modB", "modC"}) 

653 self.assertEqual(pipelineSubset1.labeled_subsets.keys(), {"subset1"}) 

654 # verify that creating a pipeline subset with the edit behavior 

655 # edits any labeled subset that contains a label not in the set of 

656 # all task labels. 

657 pipelineSubset2 = pipeline.subset_from_labels({"modA", "modB", "modC"}, PipelineSubsetCtrl.EDIT) 

658 self.assertEqual(pipelineSubset2.labeled_subsets.keys(), {"subset1", "subset2"}) 

659 self.assertEqual(pipelineSubset2.labeled_subsets["subset2"].subset, {"modC"}) 

660 

661 def testInstrument(self): 

662 # Verify that if instrument is defined it is parsed out 

663 pipeline_str = textwrap.dedent( 

664 """ 

665 description: Test Pipeline 

666 instrument: dummyCam 

667 tasks: 

668 modA: test.moduleA 

669 """ 

670 ) 

671 

672 pipeline = PipelineIR.from_string(pipeline_str) 

673 self.assertEqual(pipeline.instrument, "dummyCam") 

674 

675 def testReadTaskConfig(self): 

676 # Verify that a task with a config is read in correctly 

677 pipeline_str = textwrap.dedent( 

678 """ 

679 description: Test Pipeline 

680 tasks: 

681 modA: 

682 class: test.moduleA 

683 config: 

684 propertyA: 6 

685 propertyB: 7 

686 file: testfile.py 

687 python: "config.testDict['a'] = 9" 

688 """ 

689 ) 

690 

691 pipeline = PipelineIR.from_string(pipeline_str) 

692 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

693 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

694 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

695 

696 # Verify that multiple files are read fine 

697 pipeline_str = textwrap.dedent( 

698 """ 

699 description: Test Pipeline 

700 tasks: 

701 modA: 

702 class: test.moduleA 

703 config: 

704 file: 

705 - testfile.py 

706 - otherFile.py 

707 """ 

708 ) 

709 

710 pipeline = PipelineIR.from_string(pipeline_str) 

711 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

712 

713 # Test reading multiple Config entries 

714 pipeline_str = textwrap.dedent( 

715 """ 

716 description: Test Pipeline 

717 tasks: 

718 modA: 

719 class: test.moduleA 

720 config: 

721 - propertyA: 6 

722 propertyB: 7 

723 dataId: {"visit": 6} 

724 - propertyA: 8 

725 propertyB: 9 

726 """ 

727 ) 

728 

729 pipeline = PipelineIR.from_string(pipeline_str) 

730 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

731 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

732 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

733 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

734 

735 def testSerialization(self): 

736 # Test creating a pipeline, writing it to a file, reading the file 

737 pipeline_str = textwrap.dedent( 

738 """ 

739 description: Test Pipeline 

740 instrument: dummyCam 

741 imports: 

742 - location: $TESTDIR/testPipeline1.yaml 

743 instrument: None 

744 tasks: 

745 modC: 

746 class: test.moduleC 

747 config: 

748 - propertyA: 6 

749 propertyB: 7 

750 dataId: {"visit": 6} 

751 - propertyA: 8 

752 propertyB: 9 

753 modD: test.moduleD 

754 contracts: 

755 - modA.foo == modB.bar 

756 subsets: 

757 subA: 

758 - modA 

759 - modC 

760 """ 

761 ) 

762 

763 pipeline = PipelineIR.from_string(pipeline_str) 

764 

765 # Create the temp file, write and read 

766 with tempfile.NamedTemporaryFile() as tf: 

767 pipeline.write_to_uri(tf.name) 

768 loaded_pipeline = PipelineIR.from_uri(tf.name) 

769 self.assertEqual(pipeline, loaded_pipeline) 

770 

771 def testPipelineYamlLoader(self): 

772 # Tests that an exception is thrown in the case a key is used multiple 

773 # times in a given scope within a pipeline file 

774 pipeline_str = textwrap.dedent( 

775 """ 

776 description: Test Pipeline 

777 tasks: 

778 modA: test1 

779 modB: test2 

780 modA: test3 

781 """ 

782 ) 

783 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

784 

785 def testMultiLineStrings(self): 

786 """Test that multi-line strings in pipelines are written with 

787 '|' continuation-syntax instead of explicit newlines. 

788 """ 

789 pipeline_ir = PipelineIR({"description": "Line 1\nLine2\n", "tasks": {"modA": "task1"}}) 

790 string = str(pipeline_ir) 

791 self.assertIn("|", string) 

792 self.assertNotIn(r"\n", string) 

793 

794 

795class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

796 """Run file leak tests.""" 

797 

798 

799def setup_module(module): 

800 """Configure pytest.""" 

801 lsst.utils.tests.init() 

802 

803 

804if __name__ == "__main__": 

805 lsst.utils.tests.init() 

806 unittest.main()