Coverage for python/lsst/verify/tasks/commonMetrics.py: 35%

61 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-15 02:11 -0800

1# 

2# This file is part of verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Code for measuring metrics that apply to any Task. 

25""" 

26 

27__all__ = ["TimingMetricConfig", "TimingMetricTask", 

28 "MemoryMetricConfig", "MemoryMetricTask", 

29 ] 

30 

31import resource 

32import sys 

33 

34import astropy.units as u 

35 

36import lsst.pex.config as pexConfig 

37from lsst.pipe.base import NoWorkFound 

38 

39from lsst.verify import Measurement, Datum 

40from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \ 

41 MetadataMetricConfig 

42 

43 

44class TimeMethodMetricConfig(MetadataMetricConfig): 

45 """Common config fields for metrics based on 

46 `~lsst.utils.timer.timeMethod`. 

47 

48 These fields let metrics distinguish between different methods that have 

49 been decorated with `~lsst.utils.timer.timeMethod`. 

50 """ 

51 target = pexConfig.Field( 

52 dtype=str, 

53 doc="The method to profile, optionally prefixed by one or more tasks " 

54 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.") 

55 

56 

57# Expose TimingMetricConfig name because config-writers expect it 

58TimingMetricConfig = TimeMethodMetricConfig 

59 

60 

61class TimingMetricTask(MetadataMetricTask): 

62 """A Task that computes a wall-clock time using metadata produced by the 

63 `lsst.utils.timer.timeMethod` decorator. 

64 

65 Parameters 

66 ---------- 

67 args 

68 kwargs 

69 Constructor parameters are the same as for 

70 `lsst.verify.tasks.MetricTask`. 

71 """ 

72 

73 ConfigClass = TimingMetricConfig 

74 _DefaultName = "timingMetric" 

75 

76 @classmethod 

77 def getInputMetadataKeys(cls, config): 

78 """Get search strings for the metadata. 

79 

80 Parameters 

81 ---------- 

82 config : ``cls.ConfigClass`` 

83 Configuration for this task. 

84 

85 Returns 

86 ------- 

87 keys : `dict` 

88 A dictionary of keys, optionally prefixed by one or more tasks in 

89 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

90 

91 ``"StartTime"`` 

92 The key for when the target method started (`str`). 

93 ``"EndTime"`` 

94 The key for when the target method ended (`str`). 

95 ``"StartTimestamp"`` 

96 The key for an ISO 8601-compliant text string where the target 

97 method started (`str`). 

98 ``"EndTimestamp"`` 

99 The key for an ISO 8601-compliant text string where the target 

100 method ended (`str`). 

101 """ 

102 keyBase = config.target 

103 return {"StartTime": keyBase + "StartCpuTime", 

104 "EndTime": keyBase + "EndCpuTime", 

105 "StartTimestamp": keyBase + "StartUtc", 

106 "EndTimestamp": keyBase + "EndUtc", 

107 } 

108 

109 def makeMeasurement(self, timings): 

110 """Compute a wall-clock measurement from metadata provided by 

111 `lsst.utils.timer.timeMethod`. 

112 

113 Parameters 

114 ---------- 

115 timings : `dict` [`str`, any] 

116 A representation of the metadata passed to `run`. The `dict` has 

117 the following keys: 

118 

119 ``"StartTime"`` 

120 The time the target method started (`float` or `None`). 

121 ``"EndTime"`` 

122 The time the target method ended (`float` or `None`). 

123 ``"StartTimestamp"``, ``"EndTimestamp"`` 

124 The start and end timestamps, in an ISO 8601-compliant format 

125 (`str` or `None`). 

126 

127 Returns 

128 ------- 

129 measurement : `lsst.verify.Measurement` 

130 The running time of the target method. 

131 

132 Raises 

133 ------ 

134 lsst.verify.tasks.MetricComputationError 

135 Raised if the timing metadata are invalid. 

136 lsst.pipe.base.NoWorkFound 

137 Raised if no matching timing metadata found. 

138 """ 

139 if timings["StartTime"] is not None or timings["EndTime"] is not None: 

140 try: 

141 totalTime = timings["EndTime"] - timings["StartTime"] 

142 except TypeError: 

143 raise MetricComputationError("Invalid metadata") 

144 else: 

145 meas = Measurement(self.config.metricName, 

146 totalTime * u.second) 

147 meas.notes["estimator"] = "utils.timer.timeMethod" 

148 if timings["StartTimestamp"]: 

149 meas.extras["start"] = Datum(timings["StartTimestamp"]) 

150 if timings["EndTimestamp"]: 

151 meas.extras["end"] = Datum(timings["EndTimestamp"]) 

152 return meas 

153 else: 

154 raise NoWorkFound(f"Nothing to do: no timing information for {self.config.target} found.") 

155 

156 

157# Expose MemoryMetricConfig name because config-writers expect it 

158MemoryMetricConfig = TimeMethodMetricConfig 

159 

160 

161class MemoryMetricTask(MetadataMetricTask): 

162 """A Task that computes the maximum resident set size using metadata 

163 produced by the `lsst.utils.timer.timeMethod` decorator. 

164 

165 Parameters 

166 ---------- 

167 args 

168 kwargs 

169 Constructor parameters are the same as for 

170 `lsst.verify.tasks.MetricTask`. 

171 """ 

172 

173 ConfigClass = MemoryMetricConfig 

174 _DefaultName = "memoryMetric" 

175 

176 @classmethod 

177 def getInputMetadataKeys(cls, config): 

178 """Get search strings for the metadata. 

179 

180 Parameters 

181 ---------- 

182 config : ``cls.ConfigClass`` 

183 Configuration for this task. 

184 

185 Returns 

186 ------- 

187 keys : `dict` 

188 A dictionary of keys, optionally prefixed by one or more tasks in 

189 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

190 

191 ``"EndMemory"`` 

192 The key for the memory usage at the end of the method (`str`). 

193 ``"MetadataVersion"`` 

194 The key for the task-level metadata version. 

195 """ 

196 keyBase = config.target 

197 # Parse keyBase to get just the task prefix, if any; needed to 

198 # guarantee that returned keys all point to unique entries. 

199 # The following line returns a "."-terminated string if keyBase has a 

200 # task prefix, and "" otherwise. 

201 taskPrefix = "".join(keyBase.rpartition(".")[0:2]) 

202 

203 return {"EndMemory": keyBase + "EndMaxResidentSetSize", 

204 "MetadataVersion": taskPrefix + "__version__", 

205 } 

206 

207 def makeMeasurement(self, memory): 

208 """Compute a maximum resident set size measurement from metadata 

209 provided by `lsst.utils.timer.timeMethod`. 

210 

211 Parameters 

212 ---------- 

213 memory : `dict` [`str`, any] 

214 A representation of the metadata passed to `run`. Each `dict` has 

215 the following keys: 

216 

217 ``"EndMemory"`` 

218 The memory usage at the end of the method (`int` or `None`). 

219 ``"MetadataVersion"`` 

220 The version of the task metadata in which the value was stored 

221 (`int` or `None`). `None` is assumed to be version 0. 

222 

223 Returns 

224 ------- 

225 measurement : `lsst.verify.Measurement` 

226 The maximum memory usage of the target method. 

227 

228 Raises 

229 ------ 

230 lsst.verify.tasks.MetricComputationError 

231 Raised if the memory metadata are invalid. 

232 lsst.pipe.base.NoWorkFound 

233 Raised if no matching memory metadata found. 

234 """ 

235 if memory["EndMemory"] is not None: 

236 try: 

237 maxMemory = int(memory["EndMemory"]) 

238 version = memory["MetadataVersion"] \ 

239 if memory["MetadataVersion"] else 0 

240 except (ValueError, TypeError) as e: 

241 raise MetricComputationError("Invalid metadata") from e 

242 else: 

243 meas = Measurement(self.config.metricName, 

244 self._addUnits(maxMemory, version)) 

245 meas.notes['estimator'] = 'utils.timer.timeMethod' 

246 return meas 

247 else: 

248 raise NoWorkFound(f"Nothing to do: no memory information for {self.config.target} found.") 

249 

250 def _addUnits(self, memory, version): 

251 """Represent memory usage in correct units. 

252 

253 Parameters 

254 ---------- 

255 memory : `int` 

256 The memory usage as returned by `resource.getrusage`, in 

257 platform-dependent units. 

258 version : `int` 

259 The metadata version. If ``0``, ``memory`` is in platform-dependent 

260 units. If ``1`` or greater, ``memory`` is in bytes. 

261 

262 Returns 

263 ------- 

264 memory : `astropy.units.Quantity` 

265 The memory usage in absolute units. 

266 """ 

267 if version >= 1: 

268 return memory * u.byte 

269 elif sys.platform.startswith('darwin'): 

270 # MacOS uses bytes 

271 return memory * u.byte 

272 elif sys.platform.startswith('sunos') \ 

273 or sys.platform.startswith('solaris'): 

274 # Solaris and SunOS use pages 

275 return memory * resource.getpagesize() * u.byte 

276 else: 

277 # Assume Linux, which uses kibibytes 

278 return memory * u.kibibyte