Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2# This file is part of pipe_base. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22 

23import os 

24import tempfile 

25import textwrap 

26import unittest 

27 

28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR 

29import lsst.utils.tests 

30 

31 

32class ConfigIRTestCase(unittest.TestCase): 

33 """A test case for ConfigIR Objects 

34 

35 ConfigIR contains a method that is not exercised by the PipelineIR task, 

36 so it should be tested here 

37 """ 

38 

39 def setUp(self): 

40 pass 

41 

42 def tearDown(self): 

43 pass 

44 

45 def testMergeConfig(self): 

46 # Create some configs to merge 

47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], 

48 rest={"a": 1, "b": 2}) 

49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

52 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

53 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

54 config7 = ConfigIR(rest={"h": 9}) 

55 

56 # Merge configs with different dataIds, this should yield two elements 

57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

58 

59 # Merge configs with python blocks defined, this should yield two 

60 # elements 

61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

62 

63 # Merge configs with file defined, this should yield two elements 

64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

65 

66 # merge config2 into config1 

67 merge_result = list(config5.maybe_merge(config6)) 

68 self.assertEqual(len(merge_result), 1) 

69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

70 

71 # Cant merge configs with shared keys 

72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

73 

74 

75class PipelineIRTestCase(unittest.TestCase): 

76 """A test case for PipelineIR objects 

77 """ 

78 

79 def setUp(self): 

80 pass 

81 

82 def tearDown(self): 

83 pass 

84 

85 def testPipelineIRInitChecks(self): 

86 # Missing description 

87 pipeline_str = """ 

88 tasks: 

89 a: module.A 

90 """ 

91 with self.assertRaises(ValueError): 

92 PipelineIR.from_string(pipeline_str) 

93 

94 # Missing tasks 

95 pipeline_str = """ 

96 description: Test Pipeline 

97 """ 

98 with self.assertRaises(ValueError): 

99 PipelineIR.from_string(pipeline_str) 

100 

101 # This should raise a FileNotFoundError, as there are inherits defined 

102 # so the __init__ method should pass but the inherited file does not 

103 # exist 

104 pipeline_str = textwrap.dedent(""" 

105 description: Test Pipeline 

106 inherits: /dummy_pipeline.yaml 

107 """) 

108 

109 with self.assertRaises(FileNotFoundError): 

110 PipelineIR.from_string(pipeline_str) 

111 

112 def testTaskParsing(self): 

113 # Should be able to parse a task defined both ways 

114 pipeline_str = textwrap.dedent(""" 

115 description: Test Pipeline 

116 tasks: 

117 modA: test.modA 

118 modB: 

119 class: test.modB 

120 """) 

121 

122 pipeline = PipelineIR.from_string(pipeline_str) 

123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

125 

126 def testInheritParsing(self): 

127 # This should raise, as the two pipelines, both define the same label 

128 pipeline_str = textwrap.dedent(""" 

129 description: Test Pipeline 

130 inherits: 

131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

133 """) 

134 

135 with self.assertRaises(ValueError): 

136 PipelineIR.from_string(pipeline_str) 

137 

138 # This should pass, as the conflicting task is excluded 

139 pipeline_str = textwrap.dedent(""" 

140 description: Test Pipeline 

141 inherits: 

142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

143 exclude: modA 

144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

145 """) 

146 pipeline = PipelineIR.from_string(pipeline_str) 

147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

148 

149 # This should pass, as the conflicting task is no in includes 

150 pipeline_str = textwrap.dedent(""" 

151 description: Test Pipeline 

152 inherits: 

153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

154 include: modB 

155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

156 """) 

157 

158 pipeline = PipelineIR.from_string(pipeline_str) 

159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

160 

161 # Test that you cant include and exclude a task 

162 pipeline_str = textwrap.dedent(""" 

163 description: Test Pipeline 

164 inherits: 

165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

166 exclude: modA 

167 include: modB 

168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

169 """) 

170 

171 with self.assertRaises(ValueError): 

172 PipelineIR.from_string(pipeline_str) 

173 

174 # Test that contracts are inherited 

175 pipeline_str = textwrap.dedent(""" 

176 description: Test Pipeline 

177 inherits: 

178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

179 """) 

180 

181 pipeline = PipelineIR.from_string(pipeline_str) 

182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

183 

184 # Test that contracts are not inherited 

185 pipeline_str = textwrap.dedent(""" 

186 description: Test Pipeline 

187 inherits: 

188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

189 importContracts: False 

190 """) 

191 

192 pipeline = PipelineIR.from_string(pipeline_str) 

193 self.assertEqual(pipeline.contracts, []) 

194 

195 # Test that configs are inherited when defining the same task again 

196 # with the same label 

197 pipeline_str = textwrap.dedent(""" 

198 description: Test Pipeline 

199 inherits: 

200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

201 tasks: 

202 modA: 

203 class: "test.moduleA" 

204 config: 

205 value2: 2 

206 """) 

207 pipeline = PipelineIR.from_string(pipeline_str) 

208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

209 

210 # Test that configs are not inherited when redefining the task 

211 # associated with a label 

212 pipeline_str = textwrap.dedent(""" 

213 description: Test Pipeline 

214 inherits: 

215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

216 tasks: 

217 modA: 

218 class: "test.moduleAReplace" 

219 config: 

220 value2: 2 

221 """) 

222 pipeline = PipelineIR.from_string(pipeline_str) 

223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

224 

225 # Test that named subsets are inherited 

226 pipeline_str = textwrap.dedent(""" 

227 description: Test Pipeline 

228 inherits: 

229 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

230 """) 

231 pipeline = PipelineIR.from_string(pipeline_str) 

232 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

233 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"}) 

234 

235 # Test that inheriting and redeclaring a named subset works 

236 pipeline_str = textwrap.dedent(""" 

237 description: Test Pipeline 

238 inherits: 

239 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

240 tasks: 

241 modE: "test.moduleE" 

242 subsets: 

243 modSubset: 

244 - modE 

245 """) 

246 pipeline = PipelineIR.from_string(pipeline_str) 

247 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

248 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"}) 

249 

250 # Test that inheriting from two pipelines that both declare a named 

251 # subset with the same name fails 

252 pipeline_str = textwrap.dedent(""" 

253 description: Test Pipeline 

254 inherits: 

255 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

256 - $PIPE_BASE_DIR/tests/testPipeline3.yaml 

257 """) 

258 with self.assertRaises(ValueError): 

259 PipelineIR.from_string(pipeline_str) 

260 

261 # Test that inheriting a named subset that duplicates a label declared 

262 # in this pipeline fails 

263 pipeline_str = textwrap.dedent(""" 

264 description: Test Pipeline 

265 inherits: 

266 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

267 tasks: 

268 modSubset: "test.moduleE" 

269 """) 

270 with self.assertRaises(ValueError): 

271 PipelineIR.from_string(pipeline_str) 

272 

273 # Test that inheriting fails if a named subset and task label conflict 

274 pipeline_str = textwrap.dedent(""" 

275 description: Test Pipeline 

276 inherits: 

277 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

278 - $PIPE_BASE_DIR/tests/testPipeline4.yaml 

279 """) 

280 with self.assertRaises(ValueError): 

281 PipelineIR.from_string(pipeline_str) 

282 

283 def testReadParameters(self): 

284 # verify that parameters section are read in from a pipeline 

285 pipeline_str = textwrap.dedent(""" 

286 description: Test Pipeline 

287 parameters: 

288 value1: A 

289 value2: B 

290 tasks: 

291 modA: ModuleA 

292 """) 

293 pipeline = PipelineIR.from_string(pipeline_str) 

294 self.assertEqual(pipeline.parameters.mapping, {"value1": "A", "value2": "B"}) 

295 

296 def testTaskParameterLabel(self): 

297 # verify that "parameters" cannot be used as a task label 

298 pipeline_str = textwrap.dedent(""" 

299 description: Test Pipeline 

300 tasks: 

301 parameters: modA 

302 """) 

303 with self.assertRaises(ValueError): 

304 PipelineIR.from_string(pipeline_str) 

305 

306 def testParameterInheritance(self): 

307 # verify that inheriting parameters happens correctly 

308 pipeline_str = textwrap.dedent(""" 

309 description: Test Pipeline 

310 inherits: 

311 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

312 - location: $PIPE_BASE_DIR/tests/testPipeline2.yaml 

313 exclude: 

314 - modA 

315 

316 parameters: 

317 value4: valued 

318 """) 

319 pipeline = PipelineIR.from_string(pipeline_str) 

320 self.assertEqual(pipeline.parameters.mapping, {"value4": "valued", "value1": "valueNew", 

321 "value2": "valueB", 

322 "value3": "valueC"}) 

323 

324 def testParameterConfigFormatting(self): 

325 # verify that a config properly is formatted with parameters 

326 pipeline_str = textwrap.dedent(""" 

327 description: Test Pipeline 

328 parameters: 

329 value1: A 

330 tasks: 

331 modA: 

332 class: ModuleA 

333 config: 

334 testKey: parameters.value1 

335 """) 

336 pipeline = PipelineIR.from_string(pipeline_str) 

337 newConfig = pipeline.tasks['modA'].config[0].formatted(pipeline.parameters) 

338 self.assertEqual(newConfig.rest['testKey'], "A") 

339 

340 def testReadContracts(self): 

341 # Verify that contracts are read in from a pipeline 

342 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml") 

343 pipeline = PipelineIR.from_file(location) 

344 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

345 

346 # Verify that a contract message is loaded 

347 pipeline_str = textwrap.dedent(""" 

348 description: Test Pipeline 

349 tasks: 

350 modA: test.modA 

351 modB: 

352 class: test.modB 

353 contracts: 

354 - contract: modA.foo == modB.Bar 

355 msg: "Test message" 

356 """) 

357 

358 pipeline = PipelineIR.from_string(pipeline_str) 

359 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

360 

361 def testReadNamedSubsets(self): 

362 pipeline_str = textwrap.dedent(""" 

363 description: Test Pipeline 

364 tasks: 

365 modA: test.modA 

366 modB: 

367 class: test.modB 

368 modC: test.modC 

369 modD: test.modD 

370 subsets: 

371 subset1: 

372 - modA 

373 - modB 

374 subset2: 

375 subset: 

376 - modC 

377 - modD 

378 description: "A test named subset" 

379 """) 

380 pipeline = PipelineIR.from_string(pipeline_str) 

381 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"}) 

382 

383 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"}) 

384 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None) 

385 

386 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"}) 

387 self.assertEqual(pipeline.labeled_subsets["subset2"].description, 

388 "A test named subset") 

389 

390 # verify that forgetting a subset key is an error 

391 pipeline_str = textwrap.dedent(""" 

392 description: Test Pipeline 

393 tasks: 

394 modA: test.modA 

395 modB: 

396 class: test.modB 

397 modC: test.modC 

398 modD: test.modD 

399 subsets: 

400 subset2: 

401 sub: 

402 - modC 

403 - modD 

404 description: "A test named subset" 

405 """) 

406 with self.assertRaises(ValueError): 

407 PipelineIR.from_string(pipeline_str) 

408 

409 # verify putting a label in a named subset that is not in the task is 

410 # an error 

411 pipeline_str = textwrap.dedent(""" 

412 description: Test Pipeline 

413 tasks: 

414 modA: test.modA 

415 modB: 

416 class: test.modB 

417 modC: test.modC 

418 modD: test.modD 

419 subsets: 

420 subset2: 

421 - modC 

422 - modD 

423 - modE 

424 """) 

425 with self.assertRaises(ValueError): 

426 PipelineIR.from_string(pipeline_str) 

427 

428 def testInstrument(self): 

429 # Verify that if instrument is defined it is parsed out 

430 pipeline_str = textwrap.dedent(""" 

431 description: Test Pipeline 

432 instrument: dummyCam 

433 tasks: 

434 modA: test.moduleA 

435 """) 

436 

437 pipeline = PipelineIR.from_string(pipeline_str) 

438 self.assertEqual(pipeline.instrument, "dummyCam") 

439 

440 def testReadTaskConfig(self): 

441 # Verify that a task with a config is read in correctly 

442 pipeline_str = textwrap.dedent(""" 

443 description: Test Pipeline 

444 tasks: 

445 modA: 

446 class: test.moduleA 

447 config: 

448 propertyA: 6 

449 propertyB: 7 

450 file: testfile.py 

451 python: "config.testDict['a'] = 9" 

452 """) 

453 

454 pipeline = PipelineIR.from_string(pipeline_str) 

455 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

456 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

457 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

458 

459 # Verify that multiple files are read fine 

460 pipeline_str = textwrap.dedent(""" 

461 description: Test Pipeline 

462 tasks: 

463 modA: 

464 class: test.moduleA 

465 config: 

466 file: 

467 - testfile.py 

468 - otherFile.py 

469 """) 

470 

471 pipeline = PipelineIR.from_string(pipeline_str) 

472 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

473 

474 # Test reading multiple Config entries 

475 pipeline_str = textwrap.dedent(""" 

476 description: Test Pipeline 

477 tasks: 

478 modA: 

479 class: test.moduleA 

480 config: 

481 - propertyA: 6 

482 propertyB: 7 

483 dataId: {"visit": 6} 

484 - propertyA: 8 

485 propertyB: 9 

486 """) 

487 

488 pipeline = PipelineIR.from_string(pipeline_str) 

489 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

490 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

491 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

492 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

493 

494 def testSerialization(self): 

495 # Test creating a pipeline, writing it to a file, reading the file 

496 pipeline_str = textwrap.dedent(""" 

497 description: Test Pipeline 

498 instrument: dummyCam 

499 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

500 tasks: 

501 modC: 

502 class: test.moduleC 

503 config: 

504 - propertyA: 6 

505 propertyB: 7 

506 dataId: {"visit": 6} 

507 - propertyA: 8 

508 propertyB: 9 

509 modD: test.moduleD 

510 contracts: 

511 - modA.foo == modB.bar 

512 subsets: 

513 subA: 

514 - modA 

515 - modC 

516 """) 

517 

518 pipeline = PipelineIR.from_string(pipeline_str) 

519 

520 # Create the temp file, write and read 

521 with tempfile.NamedTemporaryFile() as tf: 

522 pipeline.to_file(tf.name) 

523 loaded_pipeline = PipelineIR.from_file(tf.name) 

524 self.assertEqual(pipeline, loaded_pipeline) 

525 

526 def testPipelineYamlLoader(self): 

527 # Tests that an exception is thrown in the case a key is used multiple 

528 # times in a given scope within a pipeline file 

529 pipeline_str = textwrap.dedent(""" 

530 description: Test Pipeline 

531 tasks: 

532 modA: test1 

533 modB: test2 

534 modA: test3 

535 """) 

536 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

537 

538 

539class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

540 pass 

541 

542 

543def setup_module(module): 

544 lsst.utils.tests.init() 

545 

546 

547if __name__ == "__main__": 547 ↛ 548line 547 didn't jump to line 548, because the condition on line 547 was never true

548 lsst.utils.tests.init() 

549 unittest.main()