Coverage for python/lsst/verify/tasks/commonMetrics.py: 38%

62 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-08 04:06 -0700

1# 

2# This file is part of verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Code for measuring metrics that apply to any Task. 

25""" 

26 

27__all__ = ["TimingMetricConfig", "TimingMetricTask", 

28 "MemoryMetricConfig", "MemoryMetricTask", 

29 ] 

30 

31import resource 

32import sys 

33 

34import astropy.units as u 

35 

36import lsst.pex.config as pexConfig 

37 

38from lsst.verify import Measurement, Datum 

39from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \ 

40 MetadataMetricConfig 

41 

42 

43class TimeMethodMetricConfig(MetadataMetricConfig): 

44 """Common config fields for metrics based on 

45 `~lsst.utils.timer.timeMethod`. 

46 

47 These fields let metrics distinguish between different methods that have 

48 been decorated with `~lsst.utils.timer.timeMethod`. 

49 """ 

50 target = pexConfig.Field( 

51 dtype=str, 

52 doc="The method to profile, optionally prefixed by one or more tasks " 

53 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.") 

54 

55 

56# Expose TimingMetricConfig name because config-writers expect it 

57TimingMetricConfig = TimeMethodMetricConfig 

58 

59 

60class TimingMetricTask(MetadataMetricTask): 

61 """A Task that computes a wall-clock time using metadata produced by the 

62 `lsst.utils.timer.timeMethod` decorator. 

63 

64 Parameters 

65 ---------- 

66 args 

67 kwargs 

68 Constructor parameters are the same as for 

69 `lsst.verify.tasks.MetricTask`. 

70 """ 

71 

72 ConfigClass = TimingMetricConfig 

73 _DefaultName = "timingMetric" 

74 

75 @classmethod 

76 def getInputMetadataKeys(cls, config): 

77 """Get search strings for the metadata. 

78 

79 Parameters 

80 ---------- 

81 config : ``cls.ConfigClass`` 

82 Configuration for this task. 

83 

84 Returns 

85 ------- 

86 keys : `dict` 

87 A dictionary of keys, optionally prefixed by one or more tasks in 

88 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

89 

90 ``"StartTime"`` 

91 The key for when the target method started (`str`). 

92 ``"EndTime"`` 

93 The key for when the target method ended (`str`). 

94 ``"StartTimestamp"`` 

95 The key for an ISO 8601-compliant text string where the target 

96 method started (`str`). 

97 ``"EndTimestamp"`` 

98 The key for an ISO 8601-compliant text string where the target 

99 method ended (`str`). 

100 """ 

101 keyBase = config.target 

102 return {"StartTime": keyBase + "StartCpuTime", 

103 "EndTime": keyBase + "EndCpuTime", 

104 "StartTimestamp": keyBase + "StartUtc", 

105 "EndTimestamp": keyBase + "EndUtc", 

106 } 

107 

108 def makeMeasurement(self, timings): 

109 """Compute a wall-clock measurement from metadata provided by 

110 `lsst.utils.timer.timeMethod`. 

111 

112 Parameters 

113 ---------- 

114 timings : `dict` [`str`, any] 

115 A representation of the metadata passed to `run`. The `dict` has 

116 the following keys: 

117 

118 ``"StartTime"`` 

119 The time the target method started (`float` or `None`). 

120 ``"EndTime"`` 

121 The time the target method ended (`float` or `None`). 

122 ``"StartTimestamp"``, ``"EndTimestamp"`` 

123 The start and end timestamps, in an ISO 8601-compliant format 

124 (`str` or `None`). 

125 

126 Returns 

127 ------- 

128 measurement : `lsst.verify.Measurement` or `None` 

129 The running time of the target method. 

130 

131 Raises 

132 ------ 

133 MetricComputationError 

134 Raised if the timing metadata are invalid. 

135 """ 

136 if timings["StartTime"] is not None or timings["EndTime"] is not None: 

137 try: 

138 totalTime = timings["EndTime"] - timings["StartTime"] 

139 except TypeError: 

140 raise MetricComputationError("Invalid metadata") 

141 else: 

142 meas = Measurement(self.config.metricName, 

143 totalTime * u.second) 

144 meas.notes["estimator"] = "utils.timer.timeMethod" 

145 if timings["StartTimestamp"]: 

146 meas.extras["start"] = Datum(timings["StartTimestamp"]) 

147 if timings["EndTimestamp"]: 

148 meas.extras["end"] = Datum(timings["EndTimestamp"]) 

149 return meas 

150 else: 

151 self.log.info("Nothing to do: no timing information for %s found.", 

152 self.config.target) 

153 return None 

154 

155 

156# Expose MemoryMetricConfig name because config-writers expect it 

157MemoryMetricConfig = TimeMethodMetricConfig 

158 

159 

160class MemoryMetricTask(MetadataMetricTask): 

161 """A Task that computes the maximum resident set size using metadata 

162 produced by the `lsst.utils.timer.timeMethod` decorator. 

163 

164 Parameters 

165 ---------- 

166 args 

167 kwargs 

168 Constructor parameters are the same as for 

169 `lsst.verify.tasks.MetricTask`. 

170 """ 

171 

172 ConfigClass = MemoryMetricConfig 

173 _DefaultName = "memoryMetric" 

174 

175 @classmethod 

176 def getInputMetadataKeys(cls, config): 

177 """Get search strings for the metadata. 

178 

179 Parameters 

180 ---------- 

181 config : ``cls.ConfigClass`` 

182 Configuration for this task. 

183 

184 Returns 

185 ------- 

186 keys : `dict` 

187 A dictionary of keys, optionally prefixed by one or more tasks in 

188 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

189 

190 ``"EndMemory"`` 

191 The key for the memory usage at the end of the method (`str`). 

192 ``"MetadataVersion"`` 

193 The key for the task-level metadata version. 

194 """ 

195 keyBase = config.target 

196 # Parse keyBase to get just the task prefix, if any; needed to 

197 # guarantee that returned keys all point to unique entries. 

198 # The following line returns a "."-terminated string if keyBase has a 

199 # task prefix, and "" otherwise. 

200 taskPrefix = "".join(keyBase.rpartition(".")[0:2]) 

201 

202 return {"EndMemory": keyBase + "EndMaxResidentSetSize", 

203 "MetadataVersion": taskPrefix + "__version__", 

204 } 

205 

206 def makeMeasurement(self, memory): 

207 """Compute a maximum resident set size measurement from metadata 

208 provided by `lsst.utils.timer.timeMethod`. 

209 

210 Parameters 

211 ---------- 

212 memory : `dict` [`str`, any] 

213 A representation of the metadata passed to `run`. Each `dict` has 

214 the following keys: 

215 

216 ``"EndMemory"`` 

217 The memory usage at the end of the method (`int` or `None`). 

218 ``"MetadataVersion"`` 

219 The version of the task metadata in which the value was stored 

220 (`int` or `None`). `None` is assumed to be version 0. 

221 

222 Returns 

223 ------- 

224 measurement : `lsst.verify.Measurement` or `None` 

225 The maximum memory usage of the target method. 

226 

227 Raises 

228 ------ 

229 MetricComputationError 

230 Raised if the memory metadata are invalid. 

231 """ 

232 if memory["EndMemory"] is not None: 

233 try: 

234 maxMemory = int(memory["EndMemory"]) 

235 version = memory["MetadataVersion"] \ 

236 if memory["MetadataVersion"] else 0 

237 except (ValueError, TypeError) as e: 

238 raise MetricComputationError("Invalid metadata") from e 

239 else: 

240 meas = Measurement(self.config.metricName, 

241 self._addUnits(maxMemory, version)) 

242 meas.notes['estimator'] = 'utils.timer.timeMethod' 

243 return meas 

244 else: 

245 self.log.info("Nothing to do: no memory information for %s found.", 

246 self.config.target) 

247 return None 

248 

249 def _addUnits(self, memory, version): 

250 """Represent memory usage in correct units. 

251 

252 Parameters 

253 ---------- 

254 memory : `int` 

255 The memory usage as returned by `resource.getrusage`, in 

256 platform-dependent units. 

257 version : `int` 

258 The metadata version. If ``0``, ``memory`` is in platform-dependent 

259 units. If ``1`` or greater, ``memory`` is in bytes. 

260 

261 Returns 

262 ------- 

263 memory : `astropy.units.Quantity` 

264 The memory usage in absolute units. 

265 """ 

266 if version >= 1: 

267 return memory * u.byte 

268 elif sys.platform.startswith('darwin'): 

269 # MacOS uses bytes 

270 return memory * u.byte 

271 elif sys.platform.startswith('sunos') \ 

272 or sys.platform.startswith('solaris'): 

273 # Solaris and SunOS use pages 

274 return memory * resource.getpagesize() * u.byte 

275 else: 

276 # Assume Linux, which uses kibibytes 

277 return memory * u.kibibyte