Coverage for tests/test_task.py: 39%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

157 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import logging 

23import time 

24import unittest 

25import numbers 

26 

27import lsst.utils.tests 

28import lsst.daf.base as dafBase 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32 

33 

34class AddConfig(pexConfig.Config): 

35 addend = pexConfig.Field(doc="amount to add", dtype=float, default=3.1) 

36 

37 

38class AddTask(pipeBase.Task): 

39 ConfigClass = AddConfig 

40 

41 @timeMethod 

42 def run(self, val): 

43 self.metadata.add("add", self.config.addend) 

44 return pipeBase.Struct( 

45 val=val + self.config.addend, 

46 ) 

47 

48 

49class MultConfig(pexConfig.Config): 

50 multiplicand = pexConfig.Field(doc="amount by which to multiply", dtype=float, default=2.5) 

51 

52 

53class MultTask(pipeBase.Task): 

54 ConfigClass = MultConfig 

55 

56 @timeMethod 

57 def run(self, val): 

58 self.metadata.add("mult", self.config.multiplicand) 

59 return pipeBase.Struct( 

60 val=val * self.config.multiplicand, 

61 ) 

62 

63 

64# prove that registry fields can also be used to hold subtasks 

65# by using a registry to hold MultTask 

66multRegistry = pexConfig.makeRegistry("Registry for Mult-like tasks") 

67multRegistry.register("stdMult", MultTask) 

68 

69 

70class AddMultConfig(pexConfig.Config): 

71 add = AddTask.makeField("add task") 

72 mult = multRegistry.makeField("mult task", default="stdMult") 

73 

74 

75class AddMultTask(pipeBase.Task): 

76 ConfigClass = AddMultConfig 

77 _DefaultName = "addMult" 

78 _add_module_logger_prefix = False 

79 

80 """First add, then multiply""" 

81 

82 def __init__(self, **keyArgs): 

83 pipeBase.Task.__init__(self, **keyArgs) 

84 self.makeSubtask("add") 

85 self.makeSubtask("mult") 

86 

87 @timeMethod 

88 def run(self, val): 

89 with self.timer("context"): 

90 addRet = self.add.run(val) 

91 multRet = self.mult.run(addRet.val) 

92 self.metadata.add("addmult", multRet.val) 

93 return pipeBase.Struct( 

94 val=multRet.val, 

95 ) 

96 

97 @timeMethod 

98 def failDec(self): 

99 """A method that fails with a decorator 

100 """ 

101 raise RuntimeError("failDec intentional error") 

102 

103 def failCtx(self): 

104 """A method that fails inside a context manager 

105 """ 

106 with self.timer("failCtx"): 

107 raise RuntimeError("failCtx intentional error") 

108 

109 

110class AddMultTask2(AddMultTask): 

111 """Subclass that gets an automatic logger prefix.""" 

112 _add_module_logger_prefix = True 

113 

114 

115class AddTwiceTask(AddTask): 

116 """Variant of AddTask that adds twice the addend""" 

117 

118 def run(self, val): 

119 addend = self.config.addend 

120 return pipeBase.Struct(val=val + (2 * addend)) 

121 

122 

123class TaskTestCase(unittest.TestCase): 

124 """A test case for Task 

125 """ 

126 

127 def setUp(self): 

128 self.valDict = dict() 

129 

130 def tearDown(self): 

131 self.valDict = None 

132 

133 def testBasics(self): 

134 """Test basic construction and use of a task 

135 """ 

136 for addend in (1.1, -3.5): 

137 for multiplicand in (0.9, -45.0): 

138 config = AddMultTask.ConfigClass() 

139 config.add.addend = addend 

140 config.mult["stdMult"].multiplicand = multiplicand 

141 # make sure both ways of accessing the registry work and give 

142 # the same result 

143 self.assertEqual(config.mult.active.multiplicand, multiplicand) 

144 addMultTask = AddMultTask(config=config) 

145 for val in (-1.0, 0.0, 17.5): 

146 ret = addMultTask.run(val=val) 

147 self.assertAlmostEqual(ret.val, (val + addend) * multiplicand) 

148 

149 def testNames(self): 

150 """Test getName() and getFullName() 

151 """ 

152 addMultTask = AddMultTask() 

153 self.assertEqual(addMultTask.getName(), "addMult") 

154 self.assertEqual(addMultTask.add.getName(), "add") 

155 self.assertEqual(addMultTask.mult.getName(), "mult") 

156 

157 self.assertEqual(addMultTask._name, "addMult") 

158 self.assertEqual(addMultTask.add._name, "add") 

159 self.assertEqual(addMultTask.mult._name, "mult") 

160 

161 self.assertEqual(addMultTask.getFullName(), "addMult") 

162 self.assertEqual(addMultTask.add.getFullName(), "addMult.add") 

163 self.assertEqual(addMultTask.mult.getFullName(), "addMult.mult") 

164 

165 self.assertEqual(addMultTask._fullName, "addMult") 

166 self.assertEqual(addMultTask.add._fullName, "addMult.add") 

167 self.assertEqual(addMultTask.mult._fullName, "addMult.mult") 

168 

169 def testLog(self): 

170 """Test the Task's logger 

171 """ 

172 addMultTask = AddMultTask() 

173 self.assertEqual(addMultTask.log.name, "addMult") 

174 self.assertEqual(addMultTask.add.log.name, "addMult.add") 

175 

176 log = logging.getLogger("tester") 

177 addMultTask = AddMultTask(log=log) 

178 self.assertEqual(addMultTask.log.name, "tester.addMult") 

179 self.assertEqual(addMultTask.add.log.name, "tester.addMult.add") 

180 

181 addMultTask2 = AddMultTask2() 

182 self.assertEqual(addMultTask2.log.name, "test_task.addMult") 

183 

184 def testGetFullMetadata(self): 

185 """Test getFullMetadata() 

186 """ 

187 addMultTask = AddMultTask() 

188 fullMetadata = addMultTask.getFullMetadata() 

189 self.assertIsInstance(fullMetadata.getPropertySet("addMult"), dafBase.PropertySet) 

190 self.assertIsInstance(fullMetadata.getPropertySet("addMult:add"), dafBase.PropertySet) 

191 self.assertIsInstance(fullMetadata.getPropertySet("addMult:mult"), dafBase.PropertySet) 

192 

193 def testEmptyMetadata(self): 

194 task = AddMultTask() 

195 task.run(val=1.2345) 

196 task.emptyMetadata() 

197 fullMetadata = task.getFullMetadata() 

198 self.assertEqual(fullMetadata.getPropertySet("addMult").nameCount(), 0) 

199 self.assertEqual(fullMetadata.getPropertySet("addMult:add").nameCount(), 0) 

200 self.assertEqual(fullMetadata.getPropertySet("addMult:mult").nameCount(), 0) 

201 

202 def testReplace(self): 

203 """Test replacing one subtask with another 

204 """ 

205 for addend in (1.1, -3.5): 

206 for multiplicand in (0.9, -45.0): 

207 config = AddMultTask.ConfigClass() 

208 config.add.retarget(AddTwiceTask) 

209 config.add.addend = addend 

210 config.mult["stdMult"].multiplicand = multiplicand 

211 addMultTask = AddMultTask(config=config) 

212 for val in (-1.0, 0.0, 17.5): 

213 ret = addMultTask.run(val=val) 

214 self.assertAlmostEqual(ret.val, (val + (2 * addend)) * multiplicand) 

215 

216 def testFail(self): 

217 """Test timers when the code they are timing fails 

218 """ 

219 addMultTask = AddMultTask() 

220 try: 

221 addMultTask.failDec() 

222 self.fail("Expected RuntimeError") 

223 except RuntimeError: 

224 self.assertTrue(addMultTask.metadata.exists("failDecEndCpuTime")) 

225 try: 

226 addMultTask.failCtx() 

227 self.fail("Expected RuntimeError") 

228 except RuntimeError: 

229 self.assertTrue(addMultTask.metadata.exists("failCtxEndCpuTime")) 

230 

231 def testTimeMethod(self): 

232 """Test that the timer is adding the right metadata 

233 """ 

234 addMultTask = AddMultTask() 

235 addMultTask.run(val=1.1) 

236 # Check existence and type 

237 for key, keyType in (("Utc", str), 

238 ("CpuTime", float), 

239 ("UserTime", float), 

240 ("SystemTime", float), 

241 ("MaxResidentSetSize", numbers.Integral), 

242 ("MinorPageFaults", numbers.Integral), 

243 ("MajorPageFaults", numbers.Integral), 

244 ("BlockInputs", numbers.Integral), 

245 ("BlockOutputs", numbers.Integral), 

246 ("VoluntaryContextSwitches", numbers.Integral), 

247 ("InvoluntaryContextSwitches", numbers.Integral), 

248 ): 

249 for when in ("Start", "End"): 

250 for method in ("run", "context"): 

251 name = method + when + key 

252 self.assertIn(name, addMultTask.metadata.names(), 

253 name + " is missing from task metadata") 

254 self.assertIsInstance(addMultTask.metadata.getScalar(name), keyType, 

255 f"{name} is not of the right type " 

256 f"({keyType} vs {type(addMultTask.metadata.getScalar(name))})") 

257 # Some basic sanity checks 

258 currCpuTime = time.process_time() 

259 self.assertLessEqual( 

260 addMultTask.metadata.getScalar("runStartCpuTime"), 

261 addMultTask.metadata.getScalar("runEndCpuTime"), 

262 ) 

263 self.assertLessEqual(addMultTask.metadata.getScalar("runEndCpuTime"), currCpuTime) 

264 self.assertLessEqual( 

265 addMultTask.metadata.getScalar("contextStartCpuTime"), 

266 addMultTask.metadata.getScalar("contextEndCpuTime"), 

267 ) 

268 self.assertLessEqual(addMultTask.metadata.getScalar("contextEndCpuTime"), currCpuTime) 

269 self.assertLessEqual( 

270 addMultTask.add.metadata.getScalar("runStartCpuTime"), 

271 addMultTask.metadata.getScalar("runEndCpuTime"), 

272 ) 

273 self.assertLessEqual(addMultTask.add.metadata.getScalar("runEndCpuTime"), currCpuTime) 

274 

275 

276class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

277 pass 

278 

279 

280def setup_module(module): 

281 lsst.utils.tests.init() 

282 

283 

284if __name__ == "__main__": 284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true

285 lsst.utils.tests.init() 

286 unittest.main()