Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2# This file is part of pipe_base. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22 

23import os 

24import tempfile 

25import textwrap 

26import unittest 

27 

28from lsst.pipe.base.pipelineIR import PipelineIR, ConfigIR 

29import lsst.utils.tests 

30 

31 

32class ConfigIRTestCase(unittest.TestCase): 

33 """A test case for ConfigIR Objects 

34 

35 ConfigIR contains a method that is not exercised by the PipelineIR task, 

36 so it should be tested here 

37 """ 

38 

39 def setUp(self): 

40 pass 

41 

42 def tearDown(self): 

43 pass 

44 

45 def testMergeConfig(self): 

46 # Create some configs to merge 

47 config1 = ConfigIR(python="config.foo=6", dataId={"visit": 7}, file=["test1.py"], 

48 rest={"a": 1, "b": 2}) 

49 config2 = ConfigIR(python=None, dataId=None, file=["test2.py"], rest={"c": 1, "d": 2}) 

50 config3 = ConfigIR(python="config.bar=7", dataId=None, file=["test3.py"], rest={"c": 1, "d": 2}) 

51 config4 = ConfigIR(python=None, dataId=None, file=["test4.py"], rest={"c": 3, "e": 4}) 

52 config5 = ConfigIR(rest={"f": 5, "g": 6}) 

53 config6 = ConfigIR(rest={"h": 7, "i": 8}) 

54 config7 = ConfigIR(rest={"h": 9}) 

55 

56 # Merge configs with different dataIds, this should yield two elements 

57 self.assertEqual(list(config1.maybe_merge(config2)), [config1, config2]) 

58 

59 # Merge configs with python blocks defined, this should yield two elements 

60 self.assertEqual(list(config1.maybe_merge(config3)), [config1, config3]) 

61 

62 # Merge configs with file defined, this should yield two elements 

63 self.assertEqual(list(config2.maybe_merge(config4)), [config2, config4]) 

64 

65 # merge config2 into config1 

66 merge_result = list(config5.maybe_merge(config6)) 

67 self.assertEqual(len(merge_result), 1) 

68 self.assertEqual(config5.rest, {"f": 5, "g": 6, "h": 7, "i": 8}) 

69 

70 # Cant merge configs with shared keys 

71 self.assertEqual(list(config5.maybe_merge(config7)), [config5, config7]) 

72 

73 

74class PipelineIRTestCase(unittest.TestCase): 

75 """A test case for PipelineIR objects 

76 """ 

77 

78 def setUp(self): 

79 pass 

80 

81 def tearDown(self): 

82 pass 

83 

84 def testPipelineIRInitChecks(self): 

85 # Missing description 

86 pipeline_str = """ 

87 tasks: 

88 a: module.A 

89 """ 

90 with self.assertRaises(ValueError): 

91 PipelineIR.from_string(pipeline_str) 

92 

93 # Missing tasks 

94 pipeline_str = """ 

95 description: Test Pipeline 

96 """ 

97 with self.assertRaises(ValueError): 

98 PipelineIR.from_string(pipeline_str) 

99 

100 # This should raise a FileNotFoundError, as there are inherits defined 

101 # so the __init__ method should pass but the inherited file does not 

102 # exist 

103 pipeline_str = textwrap.dedent(""" 

104 description: Test Pipeline 

105 inherits: /dummy_pipeline.yaml 

106 """) 

107 

108 with self.assertRaises(FileNotFoundError): 

109 PipelineIR.from_string(pipeline_str) 

110 

111 def testTaskParsing(self): 

112 # Should be able to parse a task defined both ways 

113 pipeline_str = textwrap.dedent(""" 

114 description: Test Pipeline 

115 tasks: 

116 modA: test.modA 

117 modB: 

118 class: test.modB 

119 """) 

120 

121 pipeline = PipelineIR.from_string(pipeline_str) 

122 self.assertEqual(list(pipeline.tasks.keys()), ["modA", "modB"]) 

123 self.assertEqual([t.klass for t in pipeline.tasks.values()], ["test.modA", "test.modB"]) 

124 

125 def testInheritParsing(self): 

126 # This should raise, as the two pipelines, both define the same label 

127 pipeline_str = textwrap.dedent(""" 

128 description: Test Pipeline 

129 inherits: 

130 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

131 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

132 """) 

133 

134 with self.assertRaises(ValueError): 

135 PipelineIR.from_string(pipeline_str) 

136 

137 # This should pass, as the conflicting task is excluded 

138 pipeline_str = textwrap.dedent(""" 

139 description: Test Pipeline 

140 inherits: 

141 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

142 exclude: modA 

143 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

144 """) 

145 pipeline = PipelineIR.from_string(pipeline_str) 

146 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

147 

148 # This should pass, as the conflicting task is no in includes 

149 pipeline_str = textwrap.dedent(""" 

150 description: Test Pipeline 

151 inherits: 

152 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

153 include: modB 

154 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

155 """) 

156 

157 pipeline = PipelineIR.from_string(pipeline_str) 

158 self.assertEqual(set(pipeline.tasks.keys()), set(["modA", "modB"])) 

159 

160 # Test that you cant include and exclude a task 

161 pipeline_str = textwrap.dedent(""" 

162 description: Test Pipeline 

163 inherits: 

164 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

165 exclude: modA 

166 include: modB 

167 - $PIPE_BASE_DIR/tests/testPipeline2.yaml 

168 """) 

169 

170 with self.assertRaises(ValueError): 

171 PipelineIR.from_string(pipeline_str) 

172 

173 # Test that contracts are inherited 

174 pipeline_str = textwrap.dedent(""" 

175 description: Test Pipeline 

176 inherits: 

177 - $PIPE_BASE_DIR/tests/testPipeline1.yaml 

178 """) 

179 

180 pipeline = PipelineIR.from_string(pipeline_str) 

181 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

182 

183 # Test that contracts are not inherited 

184 pipeline_str = textwrap.dedent(""" 

185 description: Test Pipeline 

186 inherits: 

187 - location: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

188 importContracts: False 

189 """) 

190 

191 pipeline = PipelineIR.from_string(pipeline_str) 

192 self.assertEqual(pipeline.contracts, []) 

193 

194 def testReadContracts(self): 

195 # Verify that contracts are read in from a pipeline 

196 location = os.path.expandvars("$PIPE_BASE_DIR/tests/testPipeline1.yaml") 

197 pipeline = PipelineIR.from_file(location) 

198 self.assertEqual(pipeline.contracts[0].contract, "modA.b == modA.c") 

199 

200 # Verify that a contract message is loaded 

201 pipeline_str = textwrap.dedent(""" 

202 description: Test Pipeline 

203 tasks: 

204 modA: test.modA 

205 modB: 

206 class: test.modB 

207 contracts: 

208 - contract: modA.foo == modB.Bar 

209 msg: "Test message" 

210 """) 

211 

212 pipeline = PipelineIR.from_string(pipeline_str) 

213 self.assertEqual(pipeline.contracts[0].msg, "Test message") 

214 

215 def testInstrument(self): 

216 # Verify that if instrument is defined it is parsed out 

217 pipeline_str = textwrap.dedent(""" 

218 description: Test Pipeline 

219 instrument: dummyCam 

220 tasks: 

221 modA: test.moduleA 

222 """) 

223 

224 pipeline = PipelineIR.from_string(pipeline_str) 

225 self.assertEqual(pipeline.instrument, "dummyCam") 

226 

227 def testReadTaskConfig(self): 

228 # Verify that a task with a config is read in correctly 

229 pipeline_str = textwrap.dedent(""" 

230 description: Test Pipeline 

231 tasks: 

232 modA: 

233 class: test.moduleA 

234 config: 

235 propertyA: 6 

236 propertyB: 7 

237 file: testfile.py 

238 python: "config.testDict['a'] = 9" 

239 """) 

240 

241 pipeline = PipelineIR.from_string(pipeline_str) 

242 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py"]) 

243 self.assertEqual(pipeline.tasks["modA"].config[0].python, "config.testDict['a'] = 9") 

244 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

245 

246 # Verify that multiple files are read fine 

247 pipeline_str = textwrap.dedent(""" 

248 description: Test Pipeline 

249 tasks: 

250 modA: 

251 class: test.moduleA 

252 config: 

253 file: 

254 - testfile.py 

255 - otherFile.py 

256 """) 

257 

258 pipeline = PipelineIR.from_string(pipeline_str) 

259 self.assertEqual(pipeline.tasks["modA"].config[0].file, ["testfile.py", "otherFile.py"]) 

260 

261 # Test reading multiple Config entries 

262 pipeline_str = textwrap.dedent(""" 

263 description: Test Pipeline 

264 tasks: 

265 modA: 

266 class: test.moduleA 

267 config: 

268 - propertyA: 6 

269 propertyB: 7 

270 dataId: {"visit": 6} 

271 - propertyA: 8 

272 propertyB: 9 

273 """) 

274 

275 pipeline = PipelineIR.from_string(pipeline_str) 

276 self.assertEqual(pipeline.tasks["modA"].config[0].rest, {"propertyA": 6, "propertyB": 7}) 

277 self.assertEqual(pipeline.tasks["modA"].config[0].dataId, {"visit": 6}) 

278 self.assertEqual(pipeline.tasks["modA"].config[1].rest, {"propertyA": 8, "propertyB": 9}) 

279 self.assertEqual(pipeline.tasks["modA"].config[1].dataId, None) 

280 

281 def testSerialization(self): 

282 # Test creating a pipeline, writing it to a file, reading the file 

283 pipeline_str = textwrap.dedent(""" 

284 description: Test Pipeline 

285 instrument: dummyCam 

286 inherits: $PIPE_BASE_DIR/tests/testPipeline1.yaml 

287 tasks: 

288 modC: 

289 class: test.moduleC 

290 config: 

291 - propertyA: 6 

292 propertyB: 7 

293 dataId: {"visit": 6} 

294 - propertyA: 8 

295 propertyB: 9 

296 modD: test.moduleD 

297 contracts: 

298 - modA.foo == modB.bar 

299 """) 

300 

301 pipeline = PipelineIR.from_string(pipeline_str) 

302 

303 # Create the temp file, write and read 

304 with tempfile.NamedTemporaryFile() as tf: 

305 pipeline.to_file(tf.name) 

306 loaded_pipeline = PipelineIR.from_file(tf.name) 

307 self.assertEqual(pipeline, loaded_pipeline) 

308 

309 def testPipelineYamlLoader(self): 

310 # Tests that an exception is thrown in the case a key is used multiple 

311 # times in a given scope within a pipeline file 

312 pipeline_str = textwrap.dedent(""" 

313 description: Test Pipeline 

314 tasks: 

315 modA: test1 

316 modB: test2 

317 modA: test3 

318 """) 

319 self.assertRaises(KeyError, PipelineIR.from_string, pipeline_str) 

320 

321 

322class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

323 pass 

324 

325 

326def setup_module(module): 

327 lsst.utils.tests.init() 

328 

329 

330if __name__ == "__main__": 330 ↛ 331line 330 didn't jump to line 331, because the condition on line 330 was never true

331 lsst.utils.tests.init() 

332 unittest.main()