Coverage for tests/test_commonMetrics.py: 27%

176 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-15 02:14 -0700

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import time 

23import unittest 

24import warnings 

25 

26import astropy.units as u 

27 

28import lsst.utils.tests 

29import lsst.pipe.base.testUtils 

30from lsst.pex.config import Config 

31from lsst.pipe.base import Task 

32from lsst.utils.timer import timeMethod 

33 

34from lsst.verify import Measurement, Name 

35from lsst.verify.gen2tasks.testUtils import MetricTaskTestCase 

36from lsst.verify.tasks import MetricComputationError, TimingMetricTask, \ 

37 MemoryMetricTask 

38from lsst.verify.tasks.testUtils import MetadataMetricTestCase 

39 

40 

41class DummyTask(Task): 

42 ConfigClass = Config 

43 _DefaultName = "NotARealTask" 

44 taskLength = 0.1 

45 

46 @timeMethod 

47 def run(self): 

48 time.sleep(self.taskLength) 

49 

50 

51class TimingMetricTestSuite(MetadataMetricTestCase): 

52 @classmethod 

53 def makeTask(cls): 

54 return TimingMetricTask(config=cls._standardConfig()) 

55 

56 @staticmethod 

57 def _standardConfig(): 

58 config = TimingMetricTask.ConfigClass() 

59 config.connections.labelName = DummyTask._DefaultName 

60 config.target = DummyTask._DefaultName + ".run" 

61 config.connections.package = "verify" 

62 config.connections.metric = "DummyTime" 

63 return config 

64 

65 def setUp(self): 

66 super().setUp() 

67 self.config = TimingMetricTestSuite._standardConfig() 

68 self.metric = Name("verify.DummyTime") 

69 

70 self.scienceTask = DummyTask() 

71 self.scienceTask.run() 

72 

73 def testValid(self): 

74 result = self.task.run(self.scienceTask.getFullMetadata()) 

75 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

76 meas = result.measurement 

77 

78 self.assertIsInstance(meas, Measurement) 

79 self.assertEqual(meas.metric_name, self.metric) 

80 self.assertGreater(meas.quantity, 0.0 * u.second) 

81 self.assertLess(meas.quantity, 2 * DummyTask.taskLength * u.second) 

82 

83 def testNoMetric(self): 

84 self.config.connections.package = "foo.bar" 

85 self.config.connections.metric = "FooBarTime" 

86 task = TimingMetricTask(config=self.config) 

87 with self.assertRaises(TypeError): 

88 task.run(self.scienceTask.getFullMetadata()) 

89 

90 def testMissingData(self): 

91 result = self.task.run(None) 

92 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

93 meas = result.measurement 

94 self.assertIsNone(meas) 

95 

96 def testRunDifferentMethod(self): 

97 self.config.target = DummyTask._DefaultName + ".runDataRef" 

98 task = TimingMetricTask(config=self.config) 

99 result = task.run(self.scienceTask.getFullMetadata()) 

100 lsst.pipe.base.testUtils.assertValidOutput(task, result) 

101 meas = result.measurement 

102 self.assertIsNone(meas) 

103 

104 def testNonsenseKeys(self): 

105 metadata = self.scienceTask.getFullMetadata() 

106 startKeys = [key 

107 for key in metadata.paramNames(topLevelOnly=False) 

108 if "StartCpuTime" in key] 

109 for key in startKeys: 

110 del metadata[key] 

111 

112 task = TimingMetricTask(config=self.config) 

113 with self.assertRaises(MetricComputationError): 

114 task.run(metadata) 

115 

116 def testBadlyTypedKeys(self): 

117 metadata = self.scienceTask.getFullMetadata() 

118 endKeys = [key 

119 for key in metadata.paramNames(topLevelOnly=False) 

120 if "EndCpuTime" in key] 

121 for key in endKeys: 

122 metadata[key] = str(float(metadata[key])) 

123 

124 task = TimingMetricTask(config=self.config) 

125 with self.assertRaises(MetricComputationError): 

126 task.run(metadata) 

127 

128 def testDeprecated(self): 

129 with warnings.catch_warnings(record=True): 

130 self.config.metric = "verify.DummyTime" 

131 self.config.connections.package = "" 

132 self.config.connections.metric = "" 

133 with warnings.catch_warnings(record=True) as emitted: 

134 self.config.validate() 

135 self.assertEqual(len(emitted), 1) 

136 self.assertEqual(emitted[0].category, FutureWarning) 

137 self.assertEqual(self.config.connections.package, "verify") 

138 self.assertEqual(self.config.connections.metric, "DummyTime") 

139 

140 

141class MemoryMetricTestSuite(MetadataMetricTestCase): 

142 @classmethod 

143 def makeTask(cls): 

144 return MemoryMetricTask(config=cls._standardConfig()) 

145 

146 @staticmethod 

147 def _standardConfig(): 

148 config = MemoryMetricTask.ConfigClass() 

149 config.connections.labelName = DummyTask._DefaultName 

150 config.target = DummyTask._DefaultName + ".run" 

151 config.connections.package = "verify" 

152 config.connections.metric = "DummyMemory" 

153 return config 

154 

155 def setUp(self): 

156 super().setUp() 

157 self.config = self._standardConfig() 

158 self.metric = Name("verify.DummyMemory") 

159 

160 self.scienceTask = DummyTask() 

161 self.scienceTask.run() 

162 

163 def testValid(self): 

164 result = self.task.run(self.scienceTask.getFullMetadata()) 

165 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

166 meas = result.measurement 

167 

168 self.assertIsInstance(meas, Measurement) 

169 self.assertEqual(meas.metric_name, self.metric) 

170 self.assertGreater(meas.quantity, 0.0 * u.byte) 

171 

172 def testNoMetric(self): 

173 self.config.connections.package = "foo.bar" 

174 self.config.connections.metric = "FooBarMemory" 

175 task = MemoryMetricTask(config=self.config) 

176 with self.assertRaises(TypeError): 

177 task.run(self.scienceTask.getFullMetadata()) 

178 

179 def testMissingData(self): 

180 result = self.task.run(None) 

181 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

182 meas = result.measurement 

183 self.assertIsNone(meas) 

184 

185 def testRunDifferentMethod(self): 

186 self.config.target = DummyTask._DefaultName + ".runDataRef" 

187 task = MemoryMetricTask(config=self.config) 

188 result = task.run(self.scienceTask.getFullMetadata()) 

189 lsst.pipe.base.testUtils.assertValidOutput(task, result) 

190 meas = result.measurement 

191 self.assertIsNone(meas) 

192 

193 def testBadlyTypedKeys(self): 

194 metadata = self.scienceTask.getFullMetadata() 

195 endKeys = [key 

196 for key in metadata.paramNames(topLevelOnly=False) 

197 if "EndMaxResidentSetSize" in key] 

198 for key in endKeys: 

199 metadata[key] = str(float(metadata[key])) 

200 

201 task = MemoryMetricTask(config=self.config) 

202 with self.assertRaises(MetricComputationError): 

203 task.run(metadata) 

204 

205 def testOldMetadata(self): 

206 """Test compatibility with version 0 metadata 

207 

208 This can't actually test differences in unit handling between version 0 

209 and version 1, but at least verifies that the code didn't choke on 

210 old-style metadata. 

211 """ 

212 newMetadata = self.scienceTask.getFullMetadata() 

213 oldMetadata = newMetadata.copy() 

214 for key in newMetadata.names(topLevelOnly=False): 

215 if "__version__" in key: 

216 oldMetadata.remove(key) 

217 

218 result = self.task.run(oldMetadata) 

219 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

220 meas = result.measurement 

221 

222 self.assertIsInstance(meas, Measurement) 

223 self.assertEqual(meas.metric_name, self.metric) 

224 

225 # Since new style is always bytes, old-style will be less or equal 

226 newResult = self.task.run(newMetadata) 

227 self.assertGreater(meas.quantity, 0.0 * u.byte) 

228 self.assertLessEqual(meas.quantity, newResult.measurement.quantity) 

229 

230 def testDeprecated(self): 

231 with warnings.catch_warnings(record=True): 

232 self.config.metric = "verify.DummyMemory" 

233 self.config.connections.package = "" 

234 self.config.connections.metric = "" 

235 with warnings.catch_warnings(record=True) as emitted: 

236 self.config.validate() 

237 self.assertEqual(len(emitted), 1) 

238 self.assertEqual(emitted[0].category, FutureWarning) 

239 self.assertEqual(self.config.connections.package, "verify") 

240 self.assertEqual(self.config.connections.metric, "DummyMemory") 

241 

242 

243# Hack around unittest's hacky test setup system 

244del MetricTaskTestCase 

245del MetadataMetricTestCase 

246 

247 

248class MemoryTester(lsst.utils.tests.MemoryTestCase): 

249 pass 

250 

251 

252def setup_module(module): 

253 lsst.utils.tests.init() 

254 

255 

256if __name__ == "__main__": 256 ↛ 257line 256 didn't jump to line 257, because the condition on line 256 was never true

257 lsst.utils.tests.init() 

258 unittest.main()