Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2# This file is part of pipe_base. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22 

23import os 

24import tempfile 

25import textwrap 

26import unittest 

27 

28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR 

29import lsst.utils.tests 

30 

31 

32class ConfigIRTestCase(unittest.TestCase): 

33 """A test case for ConfigIR Objects 

34 

35 ConfigIR contains a method that is not exercised by the PipelineIR task, 

36 so it should be tested here 

37 """ 

38 

39 def setUp(self): 

40 pass 

41 

42 def tearDown(self): 

43 pass 

44 

45 def testMergeConfig(self): 

46 # Create some configs to merge 

47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], 

48 rest={"a": 1, "b": 2}) 

49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

52 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

53 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

54 config7 = ConfigIR(rest={"h": 9}) 

55 

56 # Merge configs with different dataIds, this should yield two elements 

57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

58 

59 # Merge configs with python blocks defined, this should yield two elements 

60 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

61 

62 # Merge configs with file defined, this should yield two elements 

63 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

64 

65 # merge config2 into config1 

66 merge_result = list(config5.maybe_merge(config6)) 

67 self.assertEqual(len(merge_result), 1) 

68 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

69 

70 # Cant merge configs with shared keys 

71 self.assertEqual(list(config6.maybe_merge(config7)), [config6, config7]) 

72 

73 

74class PipelineIRTestCase(unittest.TestCase): 

75 """A test case for PipelineIR objects 

76 """ 

77 

78 def setUp(self): 

79 pass 

80 

81 def tearDown(self): 

82 pass 

83 

84 def testPipelineIRInitChecks(self): 

85 # Missing description 

86 pipeline_str = """ 

87 tasks: 

88 a: module.A 

89 """ 

90 with self.assertRaises(ValueError): 

91 PipelineIR.from_string(pipeline_str) 

92 

93 # Missing tasks 

94 pipeline_str = """ 

95 description: Test Pipeline 

96 """ 

97 with self.assertRaises(ValueError): 

98 PipelineIR.from_string(pipeline_str) 

99 

100 # This should raise a FileNotFoundError, as there are inherits defined 

101 # so the __init__ method should pass but the inherited file does not 

102 # exist 

103 pipeline_str = textwrap.dedent(""" 

104 description: Test Pipeline 

105 inherits: /dummy_pipeline.yaml 

106 """) 

107 

108 with self.assertRaises(FileNotFoundError): 

109 PipelineIR.from_string(pipeline_str) 

110 

111 def testTaskParsing(self): 

112 # Should be able to parse a task defined both ways 

113 pipeline_str = textwrap.dedent(""" 

114 description: Test Pipeline 

115 tasks: 

116 modA: test.modA 

117 modB: 

118 class: test.modB 

119 """) 

120 

121 pipeline = PipelineIR.from_string(pipeline_str) 

122 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

123 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

124 

125 def testInheritParsing(self): 

126 # This should raise, as the two pipelines, both define the same label 

127 pipeline_str = textwrap.dedent(""" 

128 description: Test Pipeline 

129 inherits: 

130 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

131 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

132 """) 

133 

134 with self.assertRaises(ValueError): 

135 PipelineIR.from_string(pipeline_str) 

136 

137 # This should pass, as the conflicting task is excluded 

138 pipeline_str = textwrap.dedent(""" 

139 description: Test Pipeline 

140 inherits: 

141 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

142 exclude: modA 

143 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

144 """) 

145 pipeline = PipelineIR.from_string(pipeline_str) 

146 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

147 

148 # This should pass, as the conflicting task is no in includes 

149 pipeline_str = textwrap.dedent(""" 

150 description: Test Pipeline 

151 inherits: 

152 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

153 include: modB 

154 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

155 """) 

156 

157 pipeline = PipelineIR.from_string(pipeline_str) 

158 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

159 

160 # Test that you cant include and exclude a task 

161 pipeline_str = textwrap.dedent(""" 

162 description: Test Pipeline 

163 inherits: 

164 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

165 exclude: modA 

166 include: modB 

167 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

168 """) 

169 

170 with self.assertRaises(ValueError): 

171 PipelineIR.from_string(pipeline_str) 

172 

173 # Test that contracts are inherited 

174 pipeline_str = textwrap.dedent(""" 

175 description: Test Pipeline 

176 inherits: 

177 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

178 """) 

179 

180 pipeline = PipelineIR.from_string(pipeline_str) 

181 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

182 

183 # Test that contracts are not inherited 

184 pipeline_str = textwrap.dedent(""" 

185 description: Test Pipeline 

186 inherits: 

187 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

188 importContracts: False 

189 """) 

190 

191 pipeline = PipelineIR.from_string(pipeline_str) 

192 self.assertEqual(pipeline.contracts, []) 

193 

194 # Test that configs are inherited when defining the same task again with 

195 # the same label 

196 pipeline_str = textwrap.dedent(""" 

197 description: Test Pipeline 

198 inherits: 

199 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

200 tasks: 

201 modA: 

202 class: "test.moduleA" 

203 config: 

204 value2: 2 

205 """) 

206 pipeline = PipelineIR.from_string(pipeline_str) 

207 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value1": 1, "value2": 2}) 

208 

209 # Test that configs are not inherited when redefining the task 

210 # associated with a label 

211 pipeline_str = textwrap.dedent(""" 

212 description: Test Pipeline 

213 inherits: 

214 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

215 tasks: 

216 modA: 

217 class: "test.moduleAReplace" 

218 config: 

219 value2: 2 

220 """) 

221 pipeline = PipelineIR.from_string(pipeline_str) 

222 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"value2": 2}) 

223 

224 def testReadContracts(self): 

225 # Verify that contracts are read in from a pipeline 

226 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml") 

227 pipeline = PipelineIR.from_file(location) 

228 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

229 

230 # Verify that a contract message is loaded 

231 pipeline_str = textwrap.dedent(""" 

232 description: Test Pipeline 

233 tasks: 

234 modA: test.modA 

235 modB: 

236 class: test.modB 

237 contracts: 

238 - contract: modA.foo == modB.Bar 

239 msg: "Test message" 

240 """) 

241 

242 pipeline = PipelineIR.from_string(pipeline_str) 

243 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

244 

245 def testInstrument(self): 

246 # Verify that if instrument is defined it is parsed out 

247 pipeline_str = textwrap.dedent(""" 

248 description: Test Pipeline 

249 instrument: dummyCam 

250 tasks: 

251 modA: test.moduleA 

252 """) 

253 

254 pipeline = PipelineIR.from_string(pipeline_str) 

255 self.assertEqual(pipeline.instrument, "dummyCam") 

256 

257 def testReadTaskConfig(self): 

258 # Verify that a task with a config is read in correctly 

259 pipeline_str = textwrap.dedent(""" 

260 description: Test Pipeline 

261 tasks: 

262 modA: 

263 class: test.moduleA 

264 config: 

265 propertyA: 6 

266 propertyB: 7 

267 file: testfile.py 

268 python: "config.testDict['a'] = 9" 

269 """) 

270 

271 pipeline = PipelineIR.from_string(pipeline_str) 

272 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

273 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

274 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

275 

276 # Verify that multiple files are read fine 

277 pipeline_str = textwrap.dedent(""" 

278 description: Test Pipeline 

279 tasks: 

280 modA: 

281 class: test.moduleA 

282 config: 

283 file: 

284 - testfile.py 

285 - otherFile.py 

286 """) 

287 

288 pipeline = PipelineIR.from_string(pipeline_str) 

289 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

290 

291 # Test reading multiple Config entries 

292 pipeline_str = textwrap.dedent(""" 

293 description: Test Pipeline 

294 tasks: 

295 modA: 

296 class: test.moduleA 

297 config: 

298 - propertyA: 6 

299 propertyB: 7 

300 dataId: {"visit": 6} 

301 - propertyA: 8 

302 propertyB: 9 

303 """) 

304 

305 pipeline = PipelineIR.from_string(pipeline_str) 

306 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

307 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

308 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

309 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

310 

311 def testSerialization(self): 

312 # Test creating a pipeline, writing it to a file, reading the file 

313 pipeline_str = textwrap.dedent(""" 

314 description: Test Pipeline 

315 instrument: dummyCam 

316 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

317 tasks: 

318 modC: 

319 class: test.moduleC 

320 config: 

321 - propertyA: 6 

322 propertyB: 7 

323 dataId: {"visit": 6} 

324 - propertyA: 8 

325 propertyB: 9 

326 modD: test.moduleD 

327 contracts: 

328 - modA.foo == modB.bar 

329 """) 

330 

331 pipeline = PipelineIR.from_string(pipeline_str) 

332 

333 # Create the temp file, write and read 

334 with tempfile.NamedTemporaryFile() as tf: 

335 pipeline.to_file(tf.name) 

336 loaded_pipeline = PipelineIR.from_file(tf.name) 

337 self.assertEqual(pipeline, loaded_pipeline) 

338 

339 def testPipelineYamlLoader(self): 

340 # Tests that an exception is thrown in the case a key is used multiple 

341 # times in a given scope within a pipeline file 

342 pipeline_str = textwrap.dedent(""" 

343 description: Test Pipeline 

344 tasks: 

345 modA: test1 

346 modB: test2 

347 modA: test3 

348 """) 

349 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

350 

351 

352class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

353 pass 

354 

355 

356def setup_module(module): 

357 lsst.utils.tests.init() 

358 

359 

360if __name__ == "__main__": 360 ↛ 361line 360 didn't jump to line 361, because the condition on line 360 was never true

361 lsst.utils.tests.init() 

362 unittest.main()