Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2# This file is part of pipe_base. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22 

23import os 

24import tempfile 

25import textwrap 

26import unittest 

27 

28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR 

29import lsst.utils.tests 

30 

31 

32class ConfigIRTestCase(unittest.TestCase): 

33 """A test case for ConfigIR Objects 

34 

35 ConfigIR contains a method that is not exercised by the PipelineIR task, 

36 so it should be tested here 

37 """ 

38 

39 def setUp(self): 

40 pass 

41 

42 def tearDown(self): 

43 pass 

44 

45 def testMergeConfig(self): 

46 # Create some configs to merge 

47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], 

48 rest={"a": 1, "b": 2}) 

49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

52 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

53 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

54 config7 = ConfigIR(rest={"h": 9}) 

55 

56 # Merge configs with different dataIds, this should yield two elements 

57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

58 

59 # Merge configs with python blocks defined, this should yield two 

60 # elements 

61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

62 

63 # Merge configs with file defined, this should yield two elements 

64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

65 

66 # merge config2 into config1 

67 merge_result = list(config5.maybe_merge(config6)) 

68 self.assertEqual(len(merge_result), 1) 

69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

70 

71 # Cant merge configs with shared keys 

72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

73 

74 

75class PipelineIRTestCase(unittest.TestCase): 

76 """A test case for PipelineIR objects 

77 """ 

78 

79 def setUp(self): 

80 pass 

81 

82 def tearDown(self): 

83 pass 

84 

85 def testPipelineIRInitChecks(self): 

86 # Missing description 

87 pipeline_str = """ 

88 tasks: 

89 a: module.A 

90 """ 

91 with self.assertRaises(ValueError): 

92 PipelineIR.from_string(pipeline_str) 

93 

94 # Missing tasks 

95 pipeline_str = """ 

96 description: Test Pipeline 

97 """ 

98 with self.assertRaises(ValueError): 

99 PipelineIR.from_string(pipeline_str) 

100 

101 # This should raise a FileNotFoundError, as there are imported defined 

102 # so the __init__ method should pass but the imported file does not 

103 # exist 

104 pipeline_str = textwrap.dedent(""" 

105 description: Test Pipeline 

106 imports: /dummy_pipeline.yaml 

107 """) 

108 

109 with self.assertRaises(FileNotFoundError): 

110 PipelineIR.from_string(pipeline_str) 

111 

112 def testTaskParsing(self): 

113 # Should be able to parse a task defined both ways 

114 pipeline_str = textwrap.dedent(""" 

115 description: Test Pipeline 

116 tasks: 

117 modA: test.modA 

118 modB: 

119 class: test.modB 

120 """) 

121 

122 pipeline = PipelineIR.from_string(pipeline_str) 

123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

125 

126 def testImportParsing(self): 

127 # This should raise, as the two pipelines, both define the same label 

128 pipeline_str = textwrap.dedent(""" 

129 description: Test Pipeline 

130 imports: 

131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

133 """) 

134 

135 with self.assertRaises(ValueError): 

136 PipelineIR.from_string(pipeline_str) 

137 

138 # This should pass, as the conflicting task is excluded 

139 pipeline_str = textwrap.dedent(""" 

140 description: Test Pipeline 

141 imports: 

142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

143 exclude: modA 

144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

145 """) 

146 pipeline = PipelineIR.from_string(pipeline_str) 

147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

148 

149 # This should pass, as the conflicting task is no in includes 

150 pipeline_str = textwrap.dedent(""" 

151 description: Test Pipeline 

152 imports: 

153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

154 include: modB 

155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

156 """) 

157 

158 pipeline = PipelineIR.from_string(pipeline_str) 

159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

160 

161 # Test that you cant include and exclude a task 

162 pipeline_str = textwrap.dedent(""" 

163 description: Test Pipeline 

164 imports: 

165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

166 exclude: modA 

167 include: modB 

168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

169 """) 

170 

171 with self.assertRaises(ValueError): 

172 PipelineIR.from_string(pipeline_str) 

173 

174 # Test that contracts are imported 

175 pipeline_str = textwrap.dedent(""" 

176 description: Test Pipeline 

177 imports: 

178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

179 """) 

180 

181 pipeline = PipelineIR.from_string(pipeline_str) 

182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

183 

184 # Test that contracts are not imported 

185 pipeline_str = textwrap.dedent(""" 

186 description: Test Pipeline 

187 imports: 

188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

189 importContracts: False 

190 """) 

191 

192 pipeline = PipelineIR.from_string(pipeline_str) 

193 self.assertEqual(pipeline.contracts, []) 

194 

195 # Test that configs are imported when defining the same task again 

196 # with the same label 

197 pipeline_str = textwrap.dedent(""" 

198 description: Test Pipeline 

199 imports: 

200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

201 tasks: 

202 modA: 

203 class: "test.moduleA" 

204 config: 

205 value2: 2 

206 """) 

207 pipeline = PipelineIR.from_string(pipeline_str) 

208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

209 

210 # Test that configs are not imported when redefining the task 

211 # associated with a label 

212 pipeline_str = textwrap.dedent(""" 

213 description: Test Pipeline 

214 imports: 

215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

216 tasks: 

217 modA: 

218 class: "test.moduleAReplace" 

219 config: 

220 value2: 2 

221 """) 

222 pipeline = PipelineIR.from_string(pipeline_str) 

223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

224 

225 # Test that named subsets are imported 

226 pipeline_str = textwrap.dedent(""" 

227 description: Test Pipeline 

228 imports: 

229 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

230 """) 

231 pipeline = PipelineIR.from_string(pipeline_str) 

232 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

233 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"}) 

234 

235 # Test that imported and redeclaring a named subset works 

236 pipeline_str = textwrap.dedent(""" 

237 description: Test Pipeline 

238 imports: 

239 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

240 tasks: 

241 modE: "test.moduleE" 

242 subsets: 

243 modSubset: 

244 - modE 

245 """) 

246 pipeline = PipelineIR.from_string(pipeline_str) 

247 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

248 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"}) 

249 

250 # Test that imported from two pipelines that both declare a named 

251 # subset with the same name fails 

252 pipeline_str = textwrap.dedent(""" 

253 description: Test Pipeline 

254 imports: 

255 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

256 - $PIPE_BASE_DIR/tests/testPipeline3.yaml 

257 """) 

258 with self.assertRaises(ValueError): 

259 PipelineIR.from_string(pipeline_str) 

260 

261 # Test that imported a named subset that duplicates a label declared 

262 # in this pipeline fails 

263 pipeline_str = textwrap.dedent(""" 

264 description: Test Pipeline 

265 imports: 

266 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

267 tasks: 

268 modSubset: "test.moduleE" 

269 """) 

270 with self.assertRaises(ValueError): 

271 PipelineIR.from_string(pipeline_str) 

272 

273 # Test that imported fails if a named subset and task label conflict 

274 pipeline_str = textwrap.dedent(""" 

275 description: Test Pipeline 

276 imports: 

277 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

278 - $PIPE_BASE_DIR/tests/testPipeline4.yaml 

279 """) 

280 with self.assertRaises(ValueError): 

281 PipelineIR.from_string(pipeline_str) 

282 

283 def testReadParameters(self): 

284 # verify that parameters section are read in from a pipeline 

285 pipeline_str = textwrap.dedent(""" 

286 description: Test Pipeline 

287 parameters: 

288 value1: A 

289 value2: B 

290 tasks: 

291 modA: ModuleA 

292 """) 

293 pipeline = PipelineIR.from_string(pipeline_str) 

294 self.assertEqual(pipeline.parameters.mapping, {"value1": "A", "value2": "B"}) 

295 

296 def testTaskParameterLabel(self): 

297 # verify that "parameters" cannot be used as a task label 

298 pipeline_str = textwrap.dedent(""" 

299 description: Test Pipeline 

300 tasks: 

301 parameters: modA 

302 """) 

303 with self.assertRaises(ValueError): 

304 PipelineIR.from_string(pipeline_str) 

305 

306 def testParameterImporting(self): 

307 # verify that importing parameters happens correctly 

308 pipeline_str = textwrap.dedent(""" 

309 description: Test Pipeline 

310 imports: 

311 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

312 - location: $PIPE_BASE_DIR/tests/testPipeline2.yaml 

313 exclude: 

314 - modA 

315 

316 parameters: 

317 value4: valued 

318 """) 

319 pipeline = PipelineIR.from_string(pipeline_str) 

320 self.assertEqual(pipeline.parameters.mapping, {"value4": "valued", "value1": "valueNew", 

321 "value2": "valueB", 

322 "value3": "valueC"}) 

323 

324 def testImportingInstrument(self): 

325 # verify an instrument is imported, or ignored, (Or otherwise modified 

326 # for potential future use) 

327 pipeline_str = textwrap.dedent(""" 

328 description: Test Pipeline 

329 imports: 

330 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

331 """) 

332 pipeline = PipelineIR.from_string(pipeline_str) 

333 self.assertEqual(pipeline.instrument, "test.instrument") 

334 

335 # verify that an imported pipeline can have its instrument set to None 

336 pipeline_str = textwrap.dedent(""" 

337 description: Test Pipeline 

338 imports: 

339 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

340 instrument: None 

341 """) 

342 pipeline = PipelineIR.from_string(pipeline_str) 

343 self.assertEqual(pipeline.instrument, None) 

344 

345 # verify that an imported pipeline can have its instrument modified 

346 pipeline_str = textwrap.dedent(""" 

347 description: Test Pipeline 

348 imports: 

349 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

350 instrument: new.instrument 

351 """) 

352 pipeline = PipelineIR.from_string(pipeline_str) 

353 self.assertEqual(pipeline.instrument, "new.instrument") 

354 

355 # Test that multiple instruments can't be defined 

356 pipeline_str = textwrap.dedent(""" 

357 description: Test Pipeline 

358 instrument: new.instrument 

359 imports: 

360 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

361 """) 

362 with self.assertRaises(ValueError): 

363 PipelineIR.from_string(pipeline_str) 

364 

365 def testParameterConfigFormatting(self): 

366 # verify that a config properly is formatted with parameters 

367 pipeline_str = textwrap.dedent(""" 

368 description: Test Pipeline 

369 parameters: 

370 value1: A 

371 tasks: 

372 modA: 

373 class: ModuleA 

374 config: 

375 testKey: parameters.value1 

376 """) 

377 pipeline = PipelineIR.from_string(pipeline_str) 

378 newConfig = pipeline.tasks['modA'].config[0].formatted(pipeline.parameters) 

379 self.assertEqual(newConfig.rest['testKey'], "A") 

380 

381 def testReadContracts(self): 

382 # Verify that contracts are read in from a pipeline 

383 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml") 

384 pipeline = PipelineIR.from_file(location) 

385 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

386 

387 # Verify that a contract message is loaded 

388 pipeline_str = textwrap.dedent(""" 

389 description: Test Pipeline 

390 tasks: 

391 modA: test.modA 

392 modB: 

393 class: test.modB 

394 contracts: 

395 - contract: modA.foo == modB.Bar 

396 msg: "Test message" 

397 """) 

398 

399 pipeline = PipelineIR.from_string(pipeline_str) 

400 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

401 

402 def testReadNamedSubsets(self): 

403 pipeline_str = textwrap.dedent(""" 

404 description: Test Pipeline 

405 tasks: 

406 modA: test.modA 

407 modB: 

408 class: test.modB 

409 modC: test.modC 

410 modD: test.modD 

411 subsets: 

412 subset1: 

413 - modA 

414 - modB 

415 subset2: 

416 subset: 

417 - modC 

418 - modD 

419 description: "A test named subset" 

420 """) 

421 pipeline = PipelineIR.from_string(pipeline_str) 

422 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"}) 

423 

424 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"}) 

425 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None) 

426 

427 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"}) 

428 self.assertEqual(pipeline.labeled_subsets["subset2"].description, 

429 "A test named subset") 

430 

431 # verify that forgetting a subset key is an error 

432 pipeline_str = textwrap.dedent(""" 

433 description: Test Pipeline 

434 tasks: 

435 modA: test.modA 

436 modB: 

437 class: test.modB 

438 modC: test.modC 

439 modD: test.modD 

440 subsets: 

441 subset2: 

442 sub: 

443 - modC 

444 - modD 

445 description: "A test named subset" 

446 """) 

447 with self.assertRaises(ValueError): 

448 PipelineIR.from_string(pipeline_str) 

449 

450 # verify putting a label in a named subset that is not in the task is 

451 # an error 

452 pipeline_str = textwrap.dedent(""" 

453 description: Test Pipeline 

454 tasks: 

455 modA: test.modA 

456 modB: 

457 class: test.modB 

458 modC: test.modC 

459 modD: test.modD 

460 subsets: 

461 subset2: 

462 - modC 

463 - modD 

464 - modE 

465 """) 

466 with self.assertRaises(ValueError): 

467 PipelineIR.from_string(pipeline_str) 

468 

469 def testInstrument(self): 

470 # Verify that if instrument is defined it is parsed out 

471 pipeline_str = textwrap.dedent(""" 

472 description: Test Pipeline 

473 instrument: dummyCam 

474 tasks: 

475 modA: test.moduleA 

476 """) 

477 

478 pipeline = PipelineIR.from_string(pipeline_str) 

479 self.assertEqual(pipeline.instrument, "dummyCam") 

480 

481 def testReadTaskConfig(self): 

482 # Verify that a task with a config is read in correctly 

483 pipeline_str = textwrap.dedent(""" 

484 description: Test Pipeline 

485 tasks: 

486 modA: 

487 class: test.moduleA 

488 config: 

489 propertyA: 6 

490 propertyB: 7 

491 file: testfile.py 

492 python: "config.testDict['a'] = 9" 

493 """) 

494 

495 pipeline = PipelineIR.from_string(pipeline_str) 

496 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

497 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

498 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

499 

500 # Verify that multiple files are read fine 

501 pipeline_str = textwrap.dedent(""" 

502 description: Test Pipeline 

503 tasks: 

504 modA: 

505 class: test.moduleA 

506 config: 

507 file: 

508 - testfile.py 

509 - otherFile.py 

510 """) 

511 

512 pipeline = PipelineIR.from_string(pipeline_str) 

513 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

514 

515 # Test reading multiple Config entries 

516 pipeline_str = textwrap.dedent(""" 

517 description: Test Pipeline 

518 tasks: 

519 modA: 

520 class: test.moduleA 

521 config: 

522 - propertyA: 6 

523 propertyB: 7 

524 dataId: {"visit": 6} 

525 - propertyA: 8 

526 propertyB: 9 

527 """) 

528 

529 pipeline = PipelineIR.from_string(pipeline_str) 

530 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

531 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

532 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

533 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

534 

535 def testSerialization(self): 

536 # Test creating a pipeline, writing it to a file, reading the file 

537 pipeline_str = textwrap.dedent(""" 

538 description: Test Pipeline 

539 instrument: dummyCam 

540 imports: 

541 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

542 instrument: None 

543 tasks: 

544 modC: 

545 class: test.moduleC 

546 config: 

547 - propertyA: 6 

548 propertyB: 7 

549 dataId: {"visit": 6} 

550 - propertyA: 8 

551 propertyB: 9 

552 modD: test.moduleD 

553 contracts: 

554 - modA.foo == modB.bar 

555 subsets: 

556 subA: 

557 - modA 

558 - modC 

559 """) 

560 

561 pipeline = PipelineIR.from_string(pipeline_str) 

562 

563 # Create the temp file, write and read 

564 with tempfile.NamedTemporaryFile() as tf: 

565 pipeline.to_file(tf.name) 

566 loaded_pipeline = PipelineIR.from_file(tf.name) 

567 self.assertEqual(pipeline, loaded_pipeline) 

568 

569 def testPipelineYamlLoader(self): 

570 # Tests that an exception is thrown in the case a key is used multiple 

571 # times in a given scope within a pipeline file 

572 pipeline_str = textwrap.dedent(""" 

573 description: Test Pipeline 

574 tasks: 

575 modA: test1 

576 modB: test2 

577 modA: test3 

578 """) 

579 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

580 

581 

582class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

583 pass 

584 

585 

586def setup_module(module): 

587 lsst.utils.tests.init() 

588 

589 

590if __name__ == "__main__": 590 ↛ 591line 590 didn't jump to line 591, because the condition on line 590 was never true

591 lsst.utils.tests.init() 

592 unittest.main()