Coverage for tests/test_commonMetrics.py: 30%

152 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-19 19:10 +0000

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import time 

23import unittest 

24 

25import astropy.units as u 

26 

27import lsst.utils.tests 

28import lsst.pipe.base.testUtils 

29from lsst.pex.config import Config 

30from lsst.pipe.base import Task 

31from lsst.utils.timer import timeMethod 

32 

33from lsst.verify import Measurement, Name 

34from lsst.verify.tasks import MetricComputationError, TimingMetricTask, \ 

35 MemoryMetricTask 

36from lsst.verify.tasks.testUtils import MetricTaskTestCase, MetadataMetricTestCase 

37 

38 

39class DummyTask(Task): 

40 ConfigClass = Config 

41 _DefaultName = "NotARealTask" 

42 taskLength = 0.1 

43 

44 @timeMethod 

45 def run(self): 

46 time.sleep(self.taskLength) 

47 

48 

49class TimingMetricTestSuite(MetadataMetricTestCase): 

50 @classmethod 

51 def makeTask(cls): 

52 return TimingMetricTask(config=cls._standardConfig()) 

53 

54 @staticmethod 

55 def _standardConfig(): 

56 config = TimingMetricTask.ConfigClass() 

57 config.connections.labelName = DummyTask._DefaultName 

58 config.target = DummyTask._DefaultName + ".run" 

59 config.connections.package = "verify" 

60 config.connections.metric = "DummyTime" 

61 return config 

62 

63 def setUp(self): 

64 super().setUp() 

65 self.config = TimingMetricTestSuite._standardConfig() 

66 self.metric = Name("verify.DummyTime") 

67 

68 self.scienceTask = DummyTask() 

69 self.scienceTask.run() 

70 

71 def testValid(self): 

72 result = self.task.run(self.scienceTask.getFullMetadata()) 

73 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

74 meas = result.measurement 

75 

76 self.assertIsInstance(meas, Measurement) 

77 self.assertEqual(meas.metric_name, self.metric) 

78 self.assertGreater(meas.quantity, 0.0 * u.second) 

79 self.assertLess(meas.quantity, 2 * DummyTask.taskLength * u.second) 

80 

81 def testNoMetric(self): 

82 self.config.connections.package = "foo.bar" 

83 self.config.connections.metric = "FooBarTime" 

84 task = TimingMetricTask(config=self.config) 

85 with self.assertRaises(TypeError): 

86 task.run(self.scienceTask.getFullMetadata()) 

87 

88 def testMissingData(self): 

89 result = self.task.run(None) 

90 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

91 meas = result.measurement 

92 self.assertIsNone(meas) 

93 

94 def testRunDifferentMethod(self): 

95 self.config.target = DummyTask._DefaultName + ".runDataRef" 

96 task = TimingMetricTask(config=self.config) 

97 result = task.run(self.scienceTask.getFullMetadata()) 

98 lsst.pipe.base.testUtils.assertValidOutput(task, result) 

99 meas = result.measurement 

100 self.assertIsNone(meas) 

101 

102 def testNonsenseKeys(self): 

103 metadata = self.scienceTask.getFullMetadata() 

104 startKeys = [key 

105 for key in metadata.paramNames(topLevelOnly=False) 

106 if "StartCpuTime" in key] 

107 for key in startKeys: 

108 del metadata[key] 

109 

110 task = TimingMetricTask(config=self.config) 

111 with self.assertRaises(MetricComputationError): 

112 task.run(metadata) 

113 

114 def testBadlyTypedKeys(self): 

115 metadata = self.scienceTask.getFullMetadata() 

116 endKeys = [key 

117 for key in metadata.paramNames(topLevelOnly=False) 

118 if "EndCpuTime" in key] 

119 for key in endKeys: 

120 metadata[key] = str(float(metadata[key])) 

121 

122 task = TimingMetricTask(config=self.config) 

123 with self.assertRaises(MetricComputationError): 

124 task.run(metadata) 

125 

126 

127class MemoryMetricTestSuite(MetadataMetricTestCase): 

128 @classmethod 

129 def makeTask(cls): 

130 return MemoryMetricTask(config=cls._standardConfig()) 

131 

132 @staticmethod 

133 def _standardConfig(): 

134 config = MemoryMetricTask.ConfigClass() 

135 config.connections.labelName = DummyTask._DefaultName 

136 config.target = DummyTask._DefaultName + ".run" 

137 config.connections.package = "verify" 

138 config.connections.metric = "DummyMemory" 

139 return config 

140 

141 def setUp(self): 

142 super().setUp() 

143 self.config = self._standardConfig() 

144 self.metric = Name("verify.DummyMemory") 

145 

146 self.scienceTask = DummyTask() 

147 self.scienceTask.run() 

148 

149 def testValid(self): 

150 result = self.task.run(self.scienceTask.getFullMetadata()) 

151 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

152 meas = result.measurement 

153 

154 self.assertIsInstance(meas, Measurement) 

155 self.assertEqual(meas.metric_name, self.metric) 

156 self.assertGreater(meas.quantity, 0.0 * u.byte) 

157 

158 def testNoMetric(self): 

159 self.config.connections.package = "foo.bar" 

160 self.config.connections.metric = "FooBarMemory" 

161 task = MemoryMetricTask(config=self.config) 

162 with self.assertRaises(TypeError): 

163 task.run(self.scienceTask.getFullMetadata()) 

164 

165 def testMissingData(self): 

166 result = self.task.run(None) 

167 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

168 meas = result.measurement 

169 self.assertIsNone(meas) 

170 

171 def testRunDifferentMethod(self): 

172 self.config.target = DummyTask._DefaultName + ".runDataRef" 

173 task = MemoryMetricTask(config=self.config) 

174 result = task.run(self.scienceTask.getFullMetadata()) 

175 lsst.pipe.base.testUtils.assertValidOutput(task, result) 

176 meas = result.measurement 

177 self.assertIsNone(meas) 

178 

179 def testBadlyTypedKeys(self): 

180 metadata = self.scienceTask.getFullMetadata() 

181 endKeys = [key 

182 for key in metadata.paramNames(topLevelOnly=False) 

183 if "EndMaxResidentSetSize" in key] 

184 for key in endKeys: 

185 metadata[key] = str(float(metadata[key])) 

186 

187 task = MemoryMetricTask(config=self.config) 

188 with self.assertRaises(MetricComputationError): 

189 task.run(metadata) 

190 

191 def testOldMetadata(self): 

192 """Test compatibility with version 0 metadata 

193 

194 This can't actually test differences in unit handling between version 0 

195 and version 1, but at least verifies that the code didn't choke on 

196 old-style metadata. 

197 """ 

198 newMetadata = self.scienceTask.getFullMetadata() 

199 oldMetadata = newMetadata.copy() 

200 for key in newMetadata.names(topLevelOnly=False): 

201 if "__version__" in key: 

202 del oldMetadata[key] 

203 

204 result = self.task.run(oldMetadata) 

205 lsst.pipe.base.testUtils.assertValidOutput(self.task, result) 

206 meas = result.measurement 

207 

208 self.assertIsInstance(meas, Measurement) 

209 self.assertEqual(meas.metric_name, self.metric) 

210 

211 # Since new style is always bytes, old-style will be less or equal 

212 newResult = self.task.run(newMetadata) 

213 self.assertGreater(meas.quantity, 0.0 * u.byte) 

214 self.assertLessEqual(meas.quantity, newResult.measurement.quantity) 

215 

216 

217# Hack around unittest's hacky test setup system 

218del MetricTaskTestCase 

219del MetadataMetricTestCase 

220 

221 

222class MemoryTester(lsst.utils.tests.MemoryTestCase): 

223 pass 

224 

225 

226def setup_module(module): 

227 lsst.utils.tests.init() 

228 

229 

230if __name__ == "__main__": 230 ↛ 231line 230 didn't jump to line 231, because the condition on line 230 was never true

231 lsst.utils.tests.init() 

232 unittest.main()