Coverage for tests/test_task.py: 34%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

189 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import logging 

23import time 

24import unittest 

25import numbers 

26import json 

27import yaml 

28 

29import lsst.utils.tests 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32from lsst.utils.timer import timeMethod 

33 

34# Whilst in transition the test can't tell which type is 

35# going to be used for metadata. 

36from lsst.pipe.base.task import _TASK_METADATA_TYPE 

37 

38 

39class AddConfig(pexConfig.Config): 

40 addend = pexConfig.Field(doc="amount to add", dtype=float, default=3.1) 

41 

42 

43class AddTask(pipeBase.Task): 

44 ConfigClass = AddConfig 

45 

46 @timeMethod 

47 def run(self, val): 

48 self.metadata.add("add", self.config.addend) 

49 return pipeBase.Struct( 

50 val=val + self.config.addend, 

51 ) 

52 

53 

54class MultConfig(pexConfig.Config): 

55 multiplicand = pexConfig.Field(doc="amount by which to multiply", dtype=float, default=2.5) 

56 

57 

58class MultTask(pipeBase.Task): 

59 ConfigClass = MultConfig 

60 

61 @timeMethod 

62 def run(self, val): 

63 self.metadata.add("mult", self.config.multiplicand) 

64 return pipeBase.Struct( 

65 val=val * self.config.multiplicand, 

66 ) 

67 

68 

69# prove that registry fields can also be used to hold subtasks 

70# by using a registry to hold MultTask 

71multRegistry = pexConfig.makeRegistry("Registry for Mult-like tasks") 

72multRegistry.register("stdMult", MultTask) 

73 

74 

75class AddMultConfig(pexConfig.Config): 

76 add = AddTask.makeField("add task") 

77 mult = multRegistry.makeField("mult task", default="stdMult") 

78 

79 

80class AddMultTask(pipeBase.Task): 

81 ConfigClass = AddMultConfig 

82 _DefaultName = "addMult" 

83 _add_module_logger_prefix = False 

84 

85 """First add, then multiply""" 

86 

87 def __init__(self, **keyArgs): 

88 pipeBase.Task.__init__(self, **keyArgs) 

89 self.makeSubtask("add") 

90 self.makeSubtask("mult") 

91 

92 @timeMethod 

93 def run(self, val): 

94 with self.timer("context"): 

95 addRet = self.add.run(val) 

96 multRet = self.mult.run(addRet.val) 

97 self.metadata.add("addmult", multRet.val) 

98 return pipeBase.Struct( 

99 val=multRet.val, 

100 ) 

101 

102 @timeMethod 

103 def failDec(self): 

104 """A method that fails with a decorator 

105 """ 

106 raise RuntimeError("failDec intentional error") 

107 

108 def failCtx(self): 

109 """A method that fails inside a context manager 

110 """ 

111 with self.timer("failCtx"): 

112 raise RuntimeError("failCtx intentional error") 

113 

114 

115class AddMultTask2(AddMultTask): 

116 """Subclass that gets an automatic logger prefix.""" 

117 _add_module_logger_prefix = True 

118 

119 

120class AddTwiceTask(AddTask): 

121 """Variant of AddTask that adds twice the addend""" 

122 

123 def run(self, val): 

124 addend = self.config.addend 

125 return pipeBase.Struct(val=val + (2 * addend)) 

126 

127 

128class TaskTestCase(unittest.TestCase): 

129 """A test case for Task 

130 """ 

131 

132 def setUp(self): 

133 self.valDict = dict() 

134 

135 def tearDown(self): 

136 self.valDict = None 

137 

138 def testBasics(self): 

139 """Test basic construction and use of a task 

140 """ 

141 for addend in (1.1, -3.5): 

142 for multiplicand in (0.9, -45.0): 

143 config = AddMultTask.ConfigClass() 

144 config.add.addend = addend 

145 config.mult["stdMult"].multiplicand = multiplicand 

146 # make sure both ways of accessing the registry work and give 

147 # the same result 

148 self.assertEqual(config.mult.active.multiplicand, multiplicand) 

149 addMultTask = AddMultTask(config=config) 

150 for val in (-1.0, 0.0, 17.5): 

151 ret = addMultTask.run(val=val) 

152 self.assertAlmostEqual(ret.val, (val + addend) * multiplicand) 

153 

154 def testNames(self): 

155 """Test getName() and getFullName() 

156 """ 

157 addMultTask = AddMultTask() 

158 self.assertEqual(addMultTask.getName(), "addMult") 

159 self.assertEqual(addMultTask.add.getName(), "add") 

160 self.assertEqual(addMultTask.mult.getName(), "mult") 

161 

162 self.assertEqual(addMultTask._name, "addMult") 

163 self.assertEqual(addMultTask.add._name, "add") 

164 self.assertEqual(addMultTask.mult._name, "mult") 

165 

166 self.assertEqual(addMultTask.getFullName(), "addMult") 

167 self.assertEqual(addMultTask.add.getFullName(), "addMult.add") 

168 self.assertEqual(addMultTask.mult.getFullName(), "addMult.mult") 

169 

170 self.assertEqual(addMultTask._fullName, "addMult") 

171 self.assertEqual(addMultTask.add._fullName, "addMult.add") 

172 self.assertEqual(addMultTask.mult._fullName, "addMult.mult") 

173 

174 def testLog(self): 

175 """Test the Task's logger 

176 """ 

177 addMultTask = AddMultTask() 

178 self.assertEqual(addMultTask.log.name, "addMult") 

179 self.assertEqual(addMultTask.add.log.name, "addMult.add") 

180 

181 log = logging.getLogger("tester") 

182 addMultTask = AddMultTask(log=log) 

183 self.assertEqual(addMultTask.log.name, "tester.addMult") 

184 self.assertEqual(addMultTask.add.log.name, "tester.addMult.add") 

185 

186 addMultTask2 = AddMultTask2() 

187 self.assertEqual(addMultTask2.log.name, f"{__name__}.addMult") 

188 

189 def testGetFullMetadata(self): 

190 """Test getFullMetadata() 

191 """ 

192 addMultTask = AddMultTask() 

193 addMultTask.run(val=1.234) # Add some metadata 

194 fullMetadata = addMultTask.getFullMetadata() 

195 self.assertIsInstance(fullMetadata["addMult"], _TASK_METADATA_TYPE) 

196 self.assertIsInstance(fullMetadata["addMult:add"], _TASK_METADATA_TYPE) 

197 self.assertIsInstance(fullMetadata["addMult:mult"], _TASK_METADATA_TYPE) 

198 self.assertEqual(set(fullMetadata), {"addMult", "addMult:add", "addMult:mult"}) 

199 

200 all_names = fullMetadata.names(topLevelOnly=False) 

201 self.assertIn("addMult", all_names) 

202 self.assertIn("addMult.runStartUtc", all_names) 

203 

204 param_names = fullMetadata.paramNames(topLevelOnly=True) 

205 # No top level keys without hierarchy 

206 self.assertEqual(set(param_names), set()) 

207 

208 param_names = fullMetadata.paramNames(topLevelOnly=False) 

209 self.assertNotIn("addMult", param_names) 

210 self.assertIn("addMult.runStartUtc", param_names) 

211 self.assertIn("addMult:add.runStartCpuTime", param_names) 

212 

213 def testEmptyMetadata(self): 

214 task = AddMultTask() 

215 task.run(val=1.2345) 

216 task.emptyMetadata() 

217 fullMetadata = task.getFullMetadata() 

218 self.assertEqual(len(fullMetadata["addMult"]), 0) 

219 self.assertEqual(len(fullMetadata["addMult:add"]), 0) 

220 self.assertEqual(len(fullMetadata["addMult:mult"]), 0) 

221 

222 def testReplace(self): 

223 """Test replacing one subtask with another 

224 """ 

225 for addend in (1.1, -3.5): 

226 for multiplicand in (0.9, -45.0): 

227 config = AddMultTask.ConfigClass() 

228 config.add.retarget(AddTwiceTask) 

229 config.add.addend = addend 

230 config.mult["stdMult"].multiplicand = multiplicand 

231 addMultTask = AddMultTask(config=config) 

232 for val in (-1.0, 0.0, 17.5): 

233 ret = addMultTask.run(val=val) 

234 self.assertAlmostEqual(ret.val, (val + (2 * addend)) * multiplicand) 

235 

236 def testFail(self): 

237 """Test timers when the code they are timing fails 

238 """ 

239 addMultTask = AddMultTask() 

240 try: 

241 addMultTask.failDec() 

242 self.fail("Expected RuntimeError") 

243 except RuntimeError: 

244 self.assertIn("failDecEndCpuTime", addMultTask.metadata) 

245 try: 

246 addMultTask.failCtx() 

247 self.fail("Expected RuntimeError") 

248 except RuntimeError: 

249 self.assertIn("failCtxEndCpuTime", addMultTask.metadata) 

250 

251 def testTimeMethod(self): 

252 """Test that the timer is adding the right metadata 

253 """ 

254 addMultTask = AddMultTask() 

255 

256 # Run twice to ensure we are additive. 

257 addMultTask.run(val=1.1) 

258 addMultTask.run(val=2.0) 

259 # Check existence and type 

260 for key, keyType in (("Utc", str), 

261 ("CpuTime", float), 

262 ("UserTime", float), 

263 ("SystemTime", float), 

264 ("MaxResidentSetSize", numbers.Integral), 

265 ("MinorPageFaults", numbers.Integral), 

266 ("MajorPageFaults", numbers.Integral), 

267 ("BlockInputs", numbers.Integral), 

268 ("BlockOutputs", numbers.Integral), 

269 ("VoluntaryContextSwitches", numbers.Integral), 

270 ("InvoluntaryContextSwitches", numbers.Integral), 

271 ): 

272 for when in ("Start", "End"): 

273 for method in ("run", "context"): 

274 name = method + when + key 

275 self.assertIn(name, addMultTask.metadata, 

276 name + " is missing from task metadata") 

277 self.assertIsInstance(addMultTask.metadata.getScalar(name), keyType, 

278 f"{name} is not of the right type " 

279 f"({keyType} vs {type(addMultTask.metadata.getScalar(name))})") 

280 # Some basic sanity checks 

281 currCpuTime = time.process_time() 

282 self.assertLessEqual( 

283 addMultTask.metadata.getScalar("runStartCpuTime"), 

284 addMultTask.metadata.getScalar("runEndCpuTime"), 

285 ) 

286 self.assertLessEqual(addMultTask.metadata.getScalar("runEndCpuTime"), currCpuTime) 

287 self.assertLessEqual( 

288 addMultTask.metadata.getScalar("contextStartCpuTime"), 

289 addMultTask.metadata.getScalar("contextEndCpuTime"), 

290 ) 

291 self.assertLessEqual(addMultTask.metadata.getScalar("contextEndCpuTime"), currCpuTime) 

292 self.assertLessEqual( 

293 addMultTask.add.metadata.getScalar("runStartCpuTime"), 

294 addMultTask.metadata.getScalar("runEndCpuTime"), 

295 ) 

296 self.assertLessEqual(addMultTask.add.metadata.getScalar("runEndCpuTime"), currCpuTime) 

297 

298 # Add some explicit values for serialization test. 

299 addMultTask.metadata["comment"] = "A comment" 

300 addMultTask.metadata["integer"] = 5 

301 addMultTask.metadata["float"] = 3.14 

302 addMultTask.metadata["bool"] = False 

303 addMultTask.metadata.add("commentList", "comment1") 

304 addMultTask.metadata.add("commentList", "comment1") 

305 addMultTask.metadata.add("intList", 6) 

306 addMultTask.metadata.add("intList", 7) 

307 addMultTask.metadata.add("boolList", False) 

308 addMultTask.metadata.add("boolList", True) 

309 addMultTask.metadata.add("floatList", 6.6) 

310 addMultTask.metadata.add("floatList", 7.8) 

311 

312 # TaskMetadata can serialize to JSON but not YAML 

313 # and PropertySet can serialize to YAML and not JSON. 

314 if hasattr(addMultTask.metadata, "json"): 

315 j = addMultTask.metadata.json() 

316 new_meta = pipeBase.TaskMetadata.parse_obj(json.loads(j)) 

317 else: 

318 y = yaml.dump(addMultTask.metadata) 

319 new_meta = yaml.safe_load(y) 

320 self.assertEqual(new_meta, addMultTask.metadata) 

321 

322 

323class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

324 pass 

325 

326 

327def setup_module(module): 

328 lsst.utils.tests.init() 

329 

330 

331if __name__ == "__main__": 331 ↛ 332line 331 didn't jump to line 332, because the condition on line 331 was never true

332 lsst.utils.tests.init() 

333 unittest.main()