Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2# This file is part of pipe_base. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22 

23import os 

24import tempfile 

25import textwrap 

26import unittest 

27 

28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR 

29import lsst.utils.tests 

30 

31 

32class ConfigIRTestCase(unittest.TestCase): 

33 """A test case for ConfigIR Objects 

34 

35 ConfigIR contains a method that is not exercised by the PipelineIR task, 

36 so it should be tested here 

37 """ 

38 

39 def setUp(self): 

40 pass 

41 

42 def tearDown(self): 

43 pass 

44 

45 def testMergeConfig(self): 

46 # Create some configs to merge 

47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], 

48 rest={"a": 1, "b": 2}) 

49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

52 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

53 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

54 config7 = ConfigIR(rest={"h": 9}) 

55 

56 # Merge configs with different dataIds, this should yield two elements 

57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

58 

59 # Merge configs with python blocks defined, this should yield two 

60 # elements 

61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

62 

63 # Merge configs with file defined, this should yield two elements 

64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

65 

66 # merge config2 into config1 

67 merge_result = list(config5.maybe_merge(config6)) 

68 self.assertEqual(len(merge_result), 1) 

69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

70 

71 # Cant merge configs with shared keys 

72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

73 

74 

75class PipelineIRTestCase(unittest.TestCase): 

76 """A test case for PipelineIR objects 

77 """ 

78 

79 def setUp(self): 

80 pass 

81 

82 def tearDown(self): 

83 pass 

84 

85 def testPipelineIRInitChecks(self): 

86 # Missing description 

87 pipeline_str = """ 

88 tasks: 

89 a: module.A 

90 """ 

91 with self.assertRaises(ValueError): 

92 PipelineIR.from_string(pipeline_str) 

93 

94 # Missing tasks 

95 pipeline_str = """ 

96 description: Test Pipeline 

97 """ 

98 with self.assertRaises(ValueError): 

99 PipelineIR.from_string(pipeline_str) 

100 

101 # This should raise a FileNotFoundError, as there are inherits defined 

102 # so the __init__ method should pass but the inherited file does not 

103 # exist 

104 pipeline_str = textwrap.dedent(""" 

105 description: Test Pipeline 

106 inherits: /dummy_pipeline.yaml 

107 """) 

108 

109 with self.assertRaises(FileNotFoundError): 

110 PipelineIR.from_string(pipeline_str) 

111 

112 def testTaskParsing(self): 

113 # Should be able to parse a task defined both ways 

114 pipeline_str = textwrap.dedent(""" 

115 description: Test Pipeline 

116 tasks: 

117 modA: test.modA 

118 modB: 

119 class: test.modB 

120 """) 

121 

122 pipeline = PipelineIR.from_string(pipeline_str) 

123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

125 

126 def testInheritParsing(self): 

127 # This should raise, as the two pipelines, both define the same label 

128 pipeline_str = textwrap.dedent(""" 

129 description: Test Pipeline 

130 inherits: 

131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

133 """) 

134 

135 with self.assertRaises(ValueError): 

136 PipelineIR.from_string(pipeline_str) 

137 

138 # This should pass, as the conflicting task is excluded 

139 pipeline_str = textwrap.dedent(""" 

140 description: Test Pipeline 

141 inherits: 

142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

143 exclude: modA 

144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

145 """) 

146 pipeline = PipelineIR.from_string(pipeline_str) 

147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

148 

149 # This should pass, as the conflicting task is no in includes 

150 pipeline_str = textwrap.dedent(""" 

151 description: Test Pipeline 

152 inherits: 

153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

154 include: modB 

155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

156 """) 

157 

158 pipeline = PipelineIR.from_string(pipeline_str) 

159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

160 

161 # Test that you cant include and exclude a task 

162 pipeline_str = textwrap.dedent(""" 

163 description: Test Pipeline 

164 inherits: 

165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

166 exclude: modA 

167 include: modB 

168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

169 """) 

170 

171 with self.assertRaises(ValueError): 

172 PipelineIR.from_string(pipeline_str) 

173 

174 # Test that contracts are inherited 

175 pipeline_str = textwrap.dedent(""" 

176 description: Test Pipeline 

177 inherits: 

178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

179 """) 

180 

181 pipeline = PipelineIR.from_string(pipeline_str) 

182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

183 

184 # Test that contracts are not inherited 

185 pipeline_str = textwrap.dedent(""" 

186 description: Test Pipeline 

187 inherits: 

188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

189 importContracts: False 

190 """) 

191 

192 pipeline = PipelineIR.from_string(pipeline_str) 

193 self.assertEqual(pipeline.contracts, []) 

194 

195 # Test that configs are inherited when defining the same task again 

196 # with the same label 

197 pipeline_str = textwrap.dedent(""" 

198 description: Test Pipeline 

199 inherits: 

200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

201 tasks: 

202 modA: 

203 class: "test.moduleA" 

204 config: 

205 value2: 2 

206 """) 

207 pipeline = PipelineIR.from_string(pipeline_str) 

208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

209 

210 # Test that configs are not inherited when redefining the task 

211 # associated with a label 

212 pipeline_str = textwrap.dedent(""" 

213 description: Test Pipeline 

214 inherits: 

215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

216 tasks: 

217 modA: 

218 class: "test.moduleAReplace" 

219 config: 

220 value2: 2 

221 """) 

222 pipeline = PipelineIR.from_string(pipeline_str) 

223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

224 

225 # Test that named subsets are inherited 

226 pipeline_str = textwrap.dedent(""" 

227 description: Test Pipeline 

228 inherits: 

229 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

230 """) 

231 pipeline = PipelineIR.from_string(pipeline_str) 

232 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

233 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modA"}) 

234 

235 # Test that inheriting and redeclaring a named subset works 

236 pipeline_str = textwrap.dedent(""" 

237 description: Test Pipeline 

238 inherits: 

239 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

240 tasks: 

241 modE: "test.moduleE" 

242 subsets: 

243 modSubset: 

244 - modE 

245 """) 

246 pipeline = PipelineIR.from_string(pipeline_str) 

247 self.assertEqual(pipeline.labeled_subsets.keys(), {"modSubset"}) 

248 self.assertEqual(pipeline.labeled_subsets["modSubset"].subset, {"modE"}) 

249 

250 # Test that inheriting from two pipelines that both declare a named 

251 # subset with the same name fails 

252 pipeline_str = textwrap.dedent(""" 

253 description: Test Pipeline 

254 inherits: 

255 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

256 - $PIPE_BASE_DIR/tests/testPipeline3.yaml 

257 """) 

258 with self.assertRaises(ValueError): 

259 PipelineIR.from_string(pipeline_str) 

260 

261 # Test that inheriting a named subset that duplicates a label declared 

262 # in this pipeline fails 

263 pipeline_str = textwrap.dedent(""" 

264 description: Test Pipeline 

265 inherits: 

266 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

267 tasks: 

268 modSubset: "test.moduleE" 

269 """) 

270 with self.assertRaises(ValueError): 

271 PipelineIR.from_string(pipeline_str) 

272 

273 # Test that inheriting fails if a named subset and task label conflict 

274 pipeline_str = textwrap.dedent(""" 

275 description: Test Pipeline 

276 inherits: 

277 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

278 - $PIPE_BASE_DIR/tests/testPipeline4.yaml 

279 """) 

280 with self.assertRaises(ValueError): 

281 PipelineIR.from_string(pipeline_str) 

282 

283 def testReadContracts(self): 

284 # Verify that contracts are read in from a pipeline 

285 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml") 

286 pipeline = PipelineIR.from_file(location) 

287 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

288 

289 # Verify that a contract message is loaded 

290 pipeline_str = textwrap.dedent(""" 

291 description: Test Pipeline 

292 tasks: 

293 modA: test.modA 

294 modB: 

295 class: test.modB 

296 contracts: 

297 - contract: modA.foo == modB.Bar 

298 msg: "Test message" 

299 """) 

300 

301 pipeline = PipelineIR.from_string(pipeline_str) 

302 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

303 

304 def testReadNamedSubsets(self): 

305 pipeline_str = textwrap.dedent(""" 

306 description: Test Pipeline 

307 tasks: 

308 modA: test.modA 

309 modB: 

310 class: test.modB 

311 modC: test.modC 

312 modD: test.modD 

313 subsets: 

314 subset1: 

315 - modA 

316 - modB 

317 subset2: 

318 subset: 

319 - modC 

320 - modD 

321 description: "A test named subset" 

322 """) 

323 pipeline = PipelineIR.from_string(pipeline_str) 

324 self.assertEqual(pipeline.labeled_subsets.keys(), {"subset1", "subset2"}) 

325 

326 self.assertEqual(pipeline.labeled_subsets["subset1"].subset, {"modA", "modB"}) 

327 self.assertEqual(pipeline.labeled_subsets["subset1"].description, None) 

328 

329 self.assertEqual(pipeline.labeled_subsets["subset2"].subset, {"modC", "modD"}) 

330 self.assertEqual(pipeline.labeled_subsets["subset2"].description, 

331 "A test named subset") 

332 

333 # verify that forgetting a subset key is an error 

334 pipeline_str = textwrap.dedent(""" 

335 description: Test Pipeline 

336 tasks: 

337 modA: test.modA 

338 modB: 

339 class: test.modB 

340 modC: test.modC 

341 modD: test.modD 

342 subsets: 

343 subset2: 

344 sub: 

345 - modC 

346 - modD 

347 description: "A test named subset" 

348 """) 

349 with self.assertRaises(ValueError): 

350 PipelineIR.from_string(pipeline_str) 

351 

352 # verify putting a label in a named subset that is not in the task is 

353 # an error 

354 pipeline_str = textwrap.dedent(""" 

355 description: Test Pipeline 

356 tasks: 

357 modA: test.modA 

358 modB: 

359 class: test.modB 

360 modC: test.modC 

361 modD: test.modD 

362 subsets: 

363 subset2: 

364 - modC 

365 - modD 

366 - modE 

367 """) 

368 with self.assertRaises(ValueError): 

369 PipelineIR.from_string(pipeline_str) 

370 

371 def testInstrument(self): 

372 # Verify that if instrument is defined it is parsed out 

373 pipeline_str = textwrap.dedent(""" 

374 description: Test Pipeline 

375 instrument: dummyCam 

376 tasks: 

377 modA: test.moduleA 

378 """) 

379 

380 pipeline = PipelineIR.from_string(pipeline_str) 

381 self.assertEqual(pipeline.instrument, "dummyCam") 

382 

383 def testReadTaskConfig(self): 

384 # Verify that a task with a config is read in correctly 

385 pipeline_str = textwrap.dedent(""" 

386 description: Test Pipeline 

387 tasks: 

388 modA: 

389 class: test.moduleA 

390 config: 

391 propertyA: 6 

392 propertyB: 7 

393 file: testfile.py 

394 python: "config.testDict['a'] = 9" 

395 """) 

396 

397 pipeline = PipelineIR.from_string(pipeline_str) 

398 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

399 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

400 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

401 

402 # Verify that multiple files are read fine 

403 pipeline_str = textwrap.dedent(""" 

404 description: Test Pipeline 

405 tasks: 

406 modA: 

407 class: test.moduleA 

408 config: 

409 file: 

410 - testfile.py 

411 - otherFile.py 

412 """) 

413 

414 pipeline = PipelineIR.from_string(pipeline_str) 

415 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

416 

417 # Test reading multiple Config entries 

418 pipeline_str = textwrap.dedent(""" 

419 description: Test Pipeline 

420 tasks: 

421 modA: 

422 class: test.moduleA 

423 config: 

424 - propertyA: 6 

425 propertyB: 7 

426 dataId: {"visit": 6} 

427 - propertyA: 8 

428 propertyB: 9 

429 """) 

430 

431 pipeline = PipelineIR.from_string(pipeline_str) 

432 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

433 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

434 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

435 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

436 

437 def testSerialization(self): 

438 # Test creating a pipeline, writing it to a file, reading the file 

439 pipeline_str = textwrap.dedent(""" 

440 description: Test Pipeline 

441 instrument: dummyCam 

442 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

443 tasks: 

444 modC: 

445 class: test.moduleC 

446 config: 

447 - propertyA: 6 

448 propertyB: 7 

449 dataId: {"visit": 6} 

450 - propertyA: 8 

451 propertyB: 9 

452 modD: test.moduleD 

453 contracts: 

454 - modA.foo == modB.bar 

455 subsets: 

456 subA: 

457 - modA 

458 - modC 

459 """) 

460 

461 pipeline = PipelineIR.from_string(pipeline_str) 

462 

463 # Create the temp file, write and read 

464 with tempfile.NamedTemporaryFile() as tf: 

465 pipeline.to_file(tf.name) 

466 loaded_pipeline = PipelineIR.from_file(tf.name) 

467 self.assertEqual(pipeline, loaded_pipeline) 

468 

469 def testPipelineYamlLoader(self): 

470 # Tests that an exception is thrown in the case a key is used multiple 

471 # times in a given scope within a pipeline file 

472 pipeline_str = textwrap.dedent(""" 

473 description: Test Pipeline 

474 tasks: 

475 modA: test1 

476 modB: test2 

477 modA: test3 

478 """) 

479 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

480 

481 

482class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

483 pass 

484 

485 

486def setup_module(module): 

487 lsst.utils.tests.init() 

488 

489 

490if __name__ == "__main__": 490 ↛ 491line 490 didn't jump to line 491, because the condition on line 490 was never true

491 lsst.utils.tests.init() 

492 unittest.main()