Coverage for tests/test_task.py: 28%

190 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-02 02:17 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import json 

23import logging 

24import numbers 

25import time 

26import unittest 

27 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.utils.tests 

31import yaml 

32 

33# Whilst in transition the test can't tell which type is 

34# going to be used for metadata. 

35from lsst.pipe.base.task import _TASK_METADATA_TYPE 

36from lsst.utils.timer import timeMethod 

37 

38 

39class AddConfig(pexConfig.Config): 

40 addend = pexConfig.Field(doc="amount to add", dtype=float, default=3.1) 

41 

42 

43class AddTask(pipeBase.Task): 

44 ConfigClass = AddConfig 

45 

46 @timeMethod 

47 def run(self, val): 

48 self.metadata.add("add", self.config.addend) 

49 return pipeBase.Struct( 

50 val=val + self.config.addend, 

51 ) 

52 

53 

54class MultConfig(pexConfig.Config): 

55 multiplicand = pexConfig.Field(doc="amount by which to multiply", dtype=float, default=2.5) 

56 

57 

58class MultTask(pipeBase.Task): 

59 ConfigClass = MultConfig 

60 

61 @timeMethod 

62 def run(self, val): 

63 self.metadata.add("mult", self.config.multiplicand) 

64 return pipeBase.Struct( 

65 val=val * self.config.multiplicand, 

66 ) 

67 

68 

69# prove that registry fields can also be used to hold subtasks 

70# by using a registry to hold MultTask 

71multRegistry = pexConfig.makeRegistry("Registry for Mult-like tasks") 

72multRegistry.register("stdMult", MultTask) 

73 

74 

75class AddMultConfig(pexConfig.Config): 

76 add = AddTask.makeField("add task") 

77 mult = multRegistry.makeField("mult task", default="stdMult") 

78 

79 

80class AddMultTask(pipeBase.Task): 

81 ConfigClass = AddMultConfig 

82 _DefaultName = "addMult" 

83 _add_module_logger_prefix = False 

84 

85 """First add, then multiply""" 

86 

87 def __init__(self, **keyArgs): 

88 pipeBase.Task.__init__(self, **keyArgs) 

89 self.makeSubtask("add") 

90 self.makeSubtask("mult") 

91 

92 @timeMethod 

93 def run(self, val): 

94 with self.timer("context"): 

95 addRet = self.add.run(val) 

96 multRet = self.mult.run(addRet.val) 

97 self.metadata.add("addmult", multRet.val) 

98 return pipeBase.Struct( 

99 val=multRet.val, 

100 ) 

101 

102 @timeMethod 

103 def failDec(self): 

104 """A method that fails with a decorator""" 

105 raise RuntimeError("failDec intentional error") 

106 

107 def failCtx(self): 

108 """A method that fails inside a context manager""" 

109 with self.timer("failCtx"): 

110 raise RuntimeError("failCtx intentional error") 

111 

112 

113class AddMultTask2(AddMultTask): 

114 """Subclass that gets an automatic logger prefix.""" 

115 

116 _add_module_logger_prefix = True 

117 

118 

119class AddTwiceTask(AddTask): 

120 """Variant of AddTask that adds twice the addend""" 

121 

122 def run(self, val): 

123 addend = self.config.addend 

124 return pipeBase.Struct(val=val + (2 * addend)) 

125 

126 

127class TaskTestCase(unittest.TestCase): 

128 """A test case for Task""" 

129 

130 def setUp(self): 

131 self.valDict = dict() 

132 

133 def tearDown(self): 

134 self.valDict = None 

135 

136 def testBasics(self): 

137 """Test basic construction and use of a task""" 

138 for addend in (1.1, -3.5): 

139 for multiplicand in (0.9, -45.0): 

140 config = AddMultTask.ConfigClass() 

141 config.add.addend = addend 

142 config.mult["stdMult"].multiplicand = multiplicand 

143 # make sure both ways of accessing the registry work and give 

144 # the same result 

145 self.assertEqual(config.mult.active.multiplicand, multiplicand) 

146 addMultTask = AddMultTask(config=config) 

147 for val in (-1.0, 0.0, 17.5): 

148 ret = addMultTask.run(val=val) 

149 self.assertAlmostEqual(ret.val, (val + addend) * multiplicand) 

150 

151 def testNames(self): 

152 """Test getName() and getFullName()""" 

153 addMultTask = AddMultTask() 

154 self.assertEqual(addMultTask.getName(), "addMult") 

155 self.assertEqual(addMultTask.add.getName(), "add") 

156 self.assertEqual(addMultTask.mult.getName(), "mult") 

157 

158 self.assertEqual(addMultTask._name, "addMult") 

159 self.assertEqual(addMultTask.add._name, "add") 

160 self.assertEqual(addMultTask.mult._name, "mult") 

161 

162 self.assertEqual(addMultTask.getFullName(), "addMult") 

163 self.assertEqual(addMultTask.add.getFullName(), "addMult.add") 

164 self.assertEqual(addMultTask.mult.getFullName(), "addMult.mult") 

165 

166 self.assertEqual(addMultTask._fullName, "addMult") 

167 self.assertEqual(addMultTask.add._fullName, "addMult.add") 

168 self.assertEqual(addMultTask.mult._fullName, "addMult.mult") 

169 

170 def testLog(self): 

171 """Test the Task's logger""" 

172 addMultTask = AddMultTask() 

173 self.assertEqual(addMultTask.log.name, "addMult") 

174 self.assertEqual(addMultTask.add.log.name, "addMult.add") 

175 

176 log = logging.getLogger("tester") 

177 addMultTask = AddMultTask(log=log) 

178 self.assertEqual(addMultTask.log.name, "tester.addMult") 

179 self.assertEqual(addMultTask.add.log.name, "tester.addMult.add") 

180 

181 addMultTask2 = AddMultTask2() 

182 self.assertEqual(addMultTask2.log.name, f"{__name__}.addMult") 

183 

184 def testGetFullMetadata(self): 

185 """Test getFullMetadata()""" 

186 addMultTask = AddMultTask() 

187 addMultTask.run(val=1.234) # Add some metadata 

188 fullMetadata = addMultTask.getFullMetadata() 

189 self.assertIsInstance(fullMetadata["addMult"], _TASK_METADATA_TYPE) 

190 self.assertIsInstance(fullMetadata["addMult:add"], _TASK_METADATA_TYPE) 

191 self.assertIsInstance(fullMetadata["addMult:mult"], _TASK_METADATA_TYPE) 

192 self.assertEqual(set(fullMetadata), {"addMult", "addMult:add", "addMult:mult"}) 

193 

194 all_names = fullMetadata.names(topLevelOnly=False) 

195 self.assertIn("addMult", all_names) 

196 self.assertIn("addMult.runStartUtc", all_names) 

197 

198 param_names = fullMetadata.paramNames(topLevelOnly=True) 

199 # No top level keys without hierarchy 

200 self.assertEqual(set(param_names), set()) 

201 

202 param_names = fullMetadata.paramNames(topLevelOnly=False) 

203 self.assertNotIn("addMult", param_names) 

204 self.assertIn("addMult.runStartUtc", param_names) 

205 self.assertIn("addMult:add.runStartCpuTime", param_names) 

206 

207 def testEmptyMetadata(self): 

208 task = AddMultTask() 

209 task.run(val=1.2345) 

210 task.emptyMetadata() 

211 fullMetadata = task.getFullMetadata() 

212 self.assertEqual(len(fullMetadata["addMult"]), 0) 

213 self.assertEqual(len(fullMetadata["addMult:add"]), 0) 

214 self.assertEqual(len(fullMetadata["addMult:mult"]), 0) 

215 

216 def testReplace(self): 

217 """Test replacing one subtask with another""" 

218 for addend in (1.1, -3.5): 

219 for multiplicand in (0.9, -45.0): 

220 config = AddMultTask.ConfigClass() 

221 config.add.retarget(AddTwiceTask) 

222 config.add.addend = addend 

223 config.mult["stdMult"].multiplicand = multiplicand 

224 addMultTask = AddMultTask(config=config) 

225 for val in (-1.0, 0.0, 17.5): 

226 ret = addMultTask.run(val=val) 

227 self.assertAlmostEqual(ret.val, (val + (2 * addend)) * multiplicand) 

228 

229 def testFail(self): 

230 """Test timers when the code they are timing fails""" 

231 addMultTask = AddMultTask() 

232 try: 

233 addMultTask.failDec() 

234 self.fail("Expected RuntimeError") 

235 except RuntimeError: 

236 self.assertIn("failDecEndCpuTime", addMultTask.metadata) 

237 try: 

238 addMultTask.failCtx() 

239 self.fail("Expected RuntimeError") 

240 except RuntimeError: 

241 self.assertIn("failCtxEndCpuTime", addMultTask.metadata) 

242 

243 def testTimeMethod(self): 

244 """Test that the timer is adding the right metadata""" 

245 addMultTask = AddMultTask() 

246 

247 # Run twice to ensure we are additive. 

248 addMultTask.run(val=1.1) 

249 addMultTask.run(val=2.0) 

250 # Check existence and type 

251 for key, keyType in ( 

252 ("Utc", str), 

253 ("CpuTime", float), 

254 ("UserTime", float), 

255 ("SystemTime", float), 

256 ("MaxResidentSetSize", numbers.Integral), 

257 ("MinorPageFaults", numbers.Integral), 

258 ("MajorPageFaults", numbers.Integral), 

259 ("BlockInputs", numbers.Integral), 

260 ("BlockOutputs", numbers.Integral), 

261 ("VoluntaryContextSwitches", numbers.Integral), 

262 ("InvoluntaryContextSwitches", numbers.Integral), 

263 ): 

264 for when in ("Start", "End"): 

265 for method in ("run", "context"): 

266 name = method + when + key 

267 self.assertIn(name, addMultTask.metadata, name + " is missing from task metadata") 

268 self.assertIsInstance( 

269 addMultTask.metadata.getScalar(name), 

270 keyType, 

271 f"{name} is not of the right type " 

272 f"({keyType} vs {type(addMultTask.metadata.getScalar(name))})", 

273 ) 

274 # Some basic sanity checks 

275 currCpuTime = time.process_time() 

276 self.assertLessEqual( 

277 addMultTask.metadata.getScalar("runStartCpuTime"), 

278 addMultTask.metadata.getScalar("runEndCpuTime"), 

279 ) 

280 self.assertLessEqual(addMultTask.metadata.getScalar("runEndCpuTime"), currCpuTime) 

281 self.assertLessEqual( 

282 addMultTask.metadata.getScalar("contextStartCpuTime"), 

283 addMultTask.metadata.getScalar("contextEndCpuTime"), 

284 ) 

285 self.assertLessEqual(addMultTask.metadata.getScalar("contextEndCpuTime"), currCpuTime) 

286 self.assertLessEqual( 

287 addMultTask.add.metadata.getScalar("runStartCpuTime"), 

288 addMultTask.metadata.getScalar("runEndCpuTime"), 

289 ) 

290 self.assertLessEqual(addMultTask.add.metadata.getScalar("runEndCpuTime"), currCpuTime) 

291 

292 # Add some explicit values for serialization test. 

293 addMultTask.metadata["comment"] = "A comment" 

294 addMultTask.metadata["integer"] = 5 

295 addMultTask.metadata["float"] = 3.14 

296 addMultTask.metadata["bool"] = False 

297 addMultTask.metadata.add("commentList", "comment1") 

298 addMultTask.metadata.add("commentList", "comment1") 

299 addMultTask.metadata.add("intList", 6) 

300 addMultTask.metadata.add("intList", 7) 

301 addMultTask.metadata.add("boolList", False) 

302 addMultTask.metadata.add("boolList", True) 

303 addMultTask.metadata.add("floatList", 6.6) 

304 addMultTask.metadata.add("floatList", 7.8) 

305 

306 # TaskMetadata can serialize to JSON but not YAML 

307 # and PropertySet can serialize to YAML and not JSON. 

308 if hasattr(addMultTask.metadata, "json"): 

309 j = addMultTask.metadata.json() 

310 new_meta = pipeBase.TaskMetadata.parse_obj(json.loads(j)) 

311 else: 

312 y = yaml.dump(addMultTask.metadata) 

313 new_meta = yaml.safe_load(y) 

314 self.assertEqual(new_meta, addMultTask.metadata) 

315 

316 

317class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

318 pass 

319 

320 

321def setup_module(module): 

322 lsst.utils.tests.init() 

323 

324 

325if __name__ == "__main__": 325 ↛ 326line 325 didn't jump to line 326, because the condition on line 325 was never true

326 lsst.utils.tests.init() 

327 unittest.main()