Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2# This file is part of pipe_base. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22 

23import os 

24import tempfile 

25import textwrap 

26import unittest 

27 

28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR 

29import lsst.utils.tests 

30 

31 

32class ConfigIRTestCase(unittest.TestCase): 

33 """A test case for ConfigIR Objects 

34 

35 ConfigIR contains a method that is not exercised by the PipelineIR task, 

36 so it should be tested here 

37 """ 

38 

39 def setUp(self): 

40 pass 

41 

42 def tearDown(self): 

43 pass 

44 

45 def testMergeConfig(self): 

46 # Create some configs to merge 

47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], 

48 rest={"a": 1, "b": 2}) 

49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

52 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

53 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

54 config7 = ConfigIR(rest={"h": 9}) 

55 

56 # Merge configs with different dataIds, this should yield two elements 

57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

58 

59 # Merge configs with python blocks defined, this should yield two 

60 # elements 

61 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

62 

63 # Merge configs with file defined, this should yield two elements 

64 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

65 

66 # merge config2 into config1 

67 merge_result = list(config5.maybe_merge(config6)) 

68 self.assertEqual(len(merge_result), 1) 

69 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

70 

71 # Cant merge configs with shared keys 

72 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

73 

74 

75class PipelineIRTestCase(unittest.TestCase): 

76 """A test case for PipelineIR objects 

77 """ 

78 

79 def setUp(self): 

80 pass 

81 

82 def tearDown(self): 

83 pass 

84 

85 def testPipelineIRInitChecks(self): 

86 # Missing description 

87 pipeline_str = """ 

88 tasks: 

89 a: module.A 

90 """ 

91 with self.assertRaises(ValueError): 

92 PipelineIR.from_string(pipeline_str) 

93 

94 # Missing tasks 

95 pipeline_str = """ 

96 description: Test Pipeline 

97 """ 

98 with self.assertRaises(ValueError): 

99 PipelineIR.from_string(pipeline_str) 

100 

101 # This should raise a FileNotFoundError, as there are inherits defined 

102 # so the __init__ method should pass but the inherited file does not 

103 # exist 

104 pipeline_str = textwrap.dedent(""" 

105 description: Test Pipeline 

106 inherits: /dummy_pipeline.yaml 

107 """) 

108 

109 with self.assertRaises(FileNotFoundError): 

110 PipelineIR.from_string(pipeline_str) 

111 

112 def testTaskParsing(self): 

113 # Should be able to parse a task defined both ways 

114 pipeline_str = textwrap.dedent(""" 

115 description: Test Pipeline 

116 tasks: 

117 modA: test.modA 

118 modB: 

119 class: test.modB 

120 """) 

121 

122 pipeline = PipelineIR.from_string(pipeline_str) 

123 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

124 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

125 

126 def testInheritParsing(self): 

127 # This should raise, as the two pipelines, both define the same label 

128 pipeline_str = textwrap.dedent(""" 

129 description: Test Pipeline 

130 inherits: 

131 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

132 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

133 """) 

134 

135 with self.assertRaises(ValueError): 

136 PipelineIR.from_string(pipeline_str) 

137 

138 # This should pass, as the conflicting task is excluded 

139 pipeline_str = textwrap.dedent(""" 

140 description: Test Pipeline 

141 inherits: 

142 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

143 exclude: modA 

144 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

145 """) 

146 pipeline = PipelineIR.from_string(pipeline_str) 

147 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

148 

149 # This should pass, as the conflicting task is no in includes 

150 pipeline_str = textwrap.dedent(""" 

151 description: Test Pipeline 

152 inherits: 

153 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

154 include: modB 

155 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

156 """) 

157 

158 pipeline = PipelineIR.from_string(pipeline_str) 

159 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

160 

161 # Test that you cant include and exclude a task 

162 pipeline_str = textwrap.dedent(""" 

163 description: Test Pipeline 

164 inherits: 

165 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

166 exclude: modA 

167 include: modB 

168 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

169 """) 

170 

171 with self.assertRaises(ValueError): 

172 PipelineIR.from_string(pipeline_str) 

173 

174 # Test that contracts are inherited 

175 pipeline_str = textwrap.dedent(""" 

176 description: Test Pipeline 

177 inherits: 

178 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

179 """) 

180 

181 pipeline = PipelineIR.from_string(pipeline_str) 

182 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

183 

184 # Test that contracts are not inherited 

185 pipeline_str = textwrap.dedent(""" 

186 description: Test Pipeline 

187 inherits: 

188 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

189 importContracts: False 

190 """) 

191 

192 pipeline = PipelineIR.from_string(pipeline_str) 

193 self.assertEqual(pipeline.contracts, []) 

194 

195 # Test that configs are inherited when defining the same task again 

196 # with the same label 

197 pipeline_str = textwrap.dedent(""" 

198 description: Test Pipeline 

199 inherits: 

200 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

201 tasks: 

202 modA: 

203 class: "test.moduleA" 

204 config: 

205 value2: 2 

206 """) 

207 pipeline = PipelineIR.from_string(pipeline_str) 

208 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

209 

210 # Test that configs are not inherited when redefining the task 

211 # associated with a label 

212 pipeline_str = textwrap.dedent(""" 

213 description: Test Pipeline 

214 inherits: 

215 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

216 tasks: 

217 modA: 

218 class: "test.moduleAReplace" 

219 config: 

220 value2: 2 

221 """) 

222 pipeline = PipelineIR.from_string(pipeline_str) 

223 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

224 

225 def testReadContracts(self): 

226 # Verify that contracts are read in from a pipeline 

227 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml") 

228 pipeline = PipelineIR.from_file(location) 

229 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

230 

231 # Verify that a contract message is loaded 

232 pipeline_str = textwrap.dedent(""" 

233 description: Test Pipeline 

234 tasks: 

235 modA: test.modA 

236 modB: 

237 class: test.modB 

238 contracts: 

239 - contract: modA.foo == modB.Bar 

240 msg: "Test message" 

241 """) 

242 

243 pipeline = PipelineIR.from_string(pipeline_str) 

244 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

245 

246 def testInstrument(self): 

247 # Verify that if instrument is defined it is parsed out 

248 pipeline_str = textwrap.dedent(""" 

249 description: Test Pipeline 

250 instrument: dummyCam 

251 tasks: 

252 modA: test.moduleA 

253 """) 

254 

255 pipeline = PipelineIR.from_string(pipeline_str) 

256 self.assertEqual(pipeline.instrument, "dummyCam") 

257 

258 def testReadTaskConfig(self): 

259 # Verify that a task with a config is read in correctly 

260 pipeline_str = textwrap.dedent(""" 

261 description: Test Pipeline 

262 tasks: 

263 modA: 

264 class: test.moduleA 

265 config: 

266 propertyA: 6 

267 propertyB: 7 

268 file: testfile.py 

269 python: "config.testDict['a'] = 9" 

270 """) 

271 

272 pipeline = PipelineIR.from_string(pipeline_str) 

273 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

274 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

275 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

276 

277 # Verify that multiple files are read fine 

278 pipeline_str = textwrap.dedent(""" 

279 description: Test Pipeline 

280 tasks: 

281 modA: 

282 class: test.moduleA 

283 config: 

284 file: 

285 - testfile.py 

286 - otherFile.py 

287 """) 

288 

289 pipeline = PipelineIR.from_string(pipeline_str) 

290 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

291 

292 # Test reading multiple Config entries 

293 pipeline_str = textwrap.dedent(""" 

294 description: Test Pipeline 

295 tasks: 

296 modA: 

297 class: test.moduleA 

298 config: 

299 - propertyA: 6 

300 propertyB: 7 

301 dataId: {"visit": 6} 

302 - propertyA: 8 

303 propertyB: 9 

304 """) 

305 

306 pipeline = PipelineIR.from_string(pipeline_str) 

307 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

308 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

309 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

310 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

311 

312 def testSerialization(self): 

313 # Test creating a pipeline, writing it to a file, reading the file 

314 pipeline_str = textwrap.dedent(""" 

315 description: Test Pipeline 

316 instrument: dummyCam 

317 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

318 tasks: 

319 modC: 

320 class: test.moduleC 

321 config: 

322 - propertyA: 6 

323 propertyB: 7 

324 dataId: {"visit": 6} 

325 - propertyA: 8 

326 propertyB: 9 

327 modD: test.moduleD 

328 contracts: 

329 - modA.foo == modB.bar 

330 """) 

331 

332 pipeline = PipelineIR.from_string(pipeline_str) 

333 

334 # Create the temp file, write and read 

335 with tempfile.NamedTemporaryFile() as tf: 

336 pipeline.to_file(tf.name) 

337 loaded_pipeline = PipelineIR.from_file(tf.name) 

338 self.assertEqual(pipeline, loaded_pipeline) 

339 

340 def testPipelineYamlLoader(self): 

341 # Tests that an exception is thrown in the case a key is used multiple 

342 # times in a given scope within a pipeline file 

343 pipeline_str = textwrap.dedent(""" 

344 description: Test Pipeline 

345 tasks: 

346 modA: test1 

347 modB: test2 

348 modA: test3 

349 """) 

350 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

351 

352 

353class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

354 pass 

355 

356 

357def setup_module(module): 

358 lsst.utils.tests.init() 

359 

360 

361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true

362 lsst.utils.tests.init() 

363 unittest.main()