Coverage for python/lsst/verify/tasks/commonMetrics.py: 34%

84 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-11 10:21 +0000

1# 

2# This file is part of verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Code for measuring metrics that apply to any Task. 

25""" 

26 

27__all__ = ["TimingMetricConfig", "TimingMetricTask", 

28 "CpuTimingMetricConfig", "CpuTimingMetricTask", 

29 "MemoryMetricConfig", "MemoryMetricTask", 

30 ] 

31 

32from datetime import datetime 

33import resource 

34import sys 

35 

36import astropy.units as u 

37 

38import lsst.pex.config as pexConfig 

39from lsst.pipe.base import NoWorkFound 

40 

41from lsst.verify import Measurement, Datum 

42from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \ 

43 MetadataMetricConfig 

44 

45 

46class TimeMethodMetricConfig(MetadataMetricConfig): 

47 """Common config fields for metrics based on 

48 `~lsst.utils.timer.timeMethod`. 

49 

50 These fields let metrics distinguish between different methods that have 

51 been decorated with `~lsst.utils.timer.timeMethod`. 

52 """ 

53 target = pexConfig.Field( 

54 dtype=str, 

55 doc="The method to profile, optionally prefixed by one or more tasks " 

56 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.") 

57 

58 

59# Expose TimingMetricConfig name because config-writers expect it 

60TimingMetricConfig = TimeMethodMetricConfig 

61 

62 

63class TimingMetricTask(MetadataMetricTask): 

64 """A Task that computes a wall-clock time using metadata produced by the 

65 `lsst.utils.timer.timeMethod` decorator. 

66 

67 Parameters 

68 ---------- 

69 args 

70 kwargs 

71 Constructor parameters are the same as for 

72 `lsst.verify.tasks.MetricTask`. 

73 """ 

74 

75 ConfigClass = TimingMetricConfig 

76 _DefaultName = "timingMetric" 

77 

78 @classmethod 

79 def getInputMetadataKeys(cls, config): 

80 """Get search strings for the metadata. 

81 

82 Parameters 

83 ---------- 

84 config : ``cls.ConfigClass`` 

85 Configuration for this task. 

86 

87 Returns 

88 ------- 

89 keys : `dict` 

90 A dictionary of keys, optionally prefixed by one or more tasks in 

91 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

92 

93 ``"StartTimestamp"`` 

94 The key for an ISO 8601-compliant text string where the target 

95 method started (`str`). 

96 ``"EndTimestamp"`` 

97 The key for an ISO 8601-compliant text string where the target 

98 method ended (`str`). 

99 """ 

100 keyBase = config.target 

101 return {"StartTimestamp": keyBase + "StartUtc", 

102 "EndTimestamp": keyBase + "EndUtc", 

103 } 

104 

105 def makeMeasurement(self, timings): 

106 """Compute a wall-clock measurement from metadata provided by 

107 `lsst.utils.timer.timeMethod`. 

108 

109 Parameters 

110 ---------- 

111 timings : `dict` [`str`, any] 

112 A representation of the metadata passed to `run`. The `dict` has 

113 the following keys: 

114 

115 ``"StartTimestamp"`` 

116 The time the target method started, in an ISO 8601-compliant 

117 format (`str` or `None`). 

118 ``"EndTimestamp"`` 

119 The time the target method ended, in an ISO 8601-compliant 

120 format (`str` or `None`). 

121 

122 Returns 

123 ------- 

124 measurement : `lsst.verify.Measurement` 

125 The running time of the target method. 

126 

127 Raises 

128 ------ 

129 lsst.verify.tasks.MetricComputationError 

130 Raised if the timing metadata are invalid. 

131 lsst.pipe.base.NoWorkFound 

132 Raised if no matching timing metadata found. 

133 """ 

134 # Use or, not and, so that unpaired keys raise MetricComputationError. 

135 if timings["StartTimestamp"] is not None or timings["EndTimestamp"] is not None: 

136 try: 

137 startTime = datetime.fromisoformat(timings["StartTimestamp"]) 

138 endTime = datetime.fromisoformat(timings["EndTimestamp"]) 

139 except (TypeError, ValueError): 

140 raise MetricComputationError("Invalid metadata") 

141 else: 

142 totalTime = (endTime - startTime).total_seconds() 

143 meas = Measurement(self.config.metricName, 

144 totalTime * u.second) 

145 meas.notes["estimator"] = "utils.timer.timeMethod" 

146 meas.extras["start"] = Datum(timings["StartTimestamp"]) 

147 meas.extras["end"] = Datum(timings["EndTimestamp"]) 

148 return meas 

149 else: 

150 raise NoWorkFound(f"Nothing to do: no timing information for {self.config.target} found.") 

151 

152 

153# Expose CpuTimingMetricConfig name because config-writers expect it 

154CpuTimingMetricConfig = TimeMethodMetricConfig 

155 

156 

157class CpuTimingMetricTask(MetadataMetricTask): 

158 """A Task that computes a CPU time using metadata produced by the 

159 `lsst.utils.timer.timeMethod` decorator. 

160 

161 Parameters 

162 ---------- 

163 args 

164 kwargs 

165 Constructor parameters are the same as for 

166 `lsst.verify.tasks.MetricTask`. 

167 """ 

168 

169 ConfigClass = CpuTimingMetricConfig 

170 _DefaultName = "cpuTimingMetric" 

171 

172 @classmethod 

173 def getInputMetadataKeys(cls, config): 

174 """Get search strings for the metadata. 

175 

176 Parameters 

177 ---------- 

178 config : ``cls.ConfigClass`` 

179 Configuration for this task. 

180 

181 Returns 

182 ------- 

183 keys : `dict` 

184 A dictionary of keys, optionally prefixed by one or more tasks in 

185 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

186 

187 ``"StartTime"`` 

188 The key for when the target method started (`str`). 

189 ``"EndTime"`` 

190 The key for when the target method ended (`str`). 

191 ``"StartTimestamp"`` 

192 The key for an ISO 8601-compliant text string where the target 

193 method started (`str`). 

194 ``"EndTimestamp"`` 

195 The key for an ISO 8601-compliant text string where the target 

196 method ended (`str`). 

197 """ 

198 keyBase = config.target 

199 return {"StartTime": keyBase + "StartCpuTime", 

200 "EndTime": keyBase + "EndCpuTime", 

201 "StartTimestamp": keyBase + "StartUtc", 

202 "EndTimestamp": keyBase + "EndUtc", 

203 } 

204 

205 def makeMeasurement(self, timings): 

206 """Compute a wall-clock measurement from metadata provided by 

207 `lsst.utils.timer.timeMethod`. 

208 

209 Parameters 

210 ---------- 

211 timings : `dict` [`str`, any] 

212 A representation of the metadata passed to `run`. The `dict` has 

213 the following keys: 

214 

215 ``"StartTime"`` 

216 The time the target method started (`float` or `None`). 

217 ``"EndTime"`` 

218 The time the target method ended (`float` or `None`). 

219 ``"StartTimestamp"``, ``"EndTimestamp"`` 

220 The start and end timestamps, in an ISO 8601-compliant format 

221 (`str` or `None`). 

222 

223 Returns 

224 ------- 

225 measurement : `lsst.verify.Measurement` 

226 The running time of the target method. 

227 

228 Raises 

229 ------ 

230 lsst.verify.tasks.MetricComputationError 

231 Raised if the timing metadata are invalid. 

232 lsst.pipe.base.NoWorkFound 

233 Raised if no matching timing metadata found. 

234 """ 

235 # Use or, not and, so that unpaired keys raise MetricComputationError. 

236 if timings["StartTime"] is not None or timings["EndTime"] is not None: 

237 try: 

238 totalTime = timings["EndTime"] - timings["StartTime"] 

239 except TypeError: 

240 raise MetricComputationError("Invalid metadata") 

241 else: 

242 meas = Measurement(self.config.metricName, 

243 totalTime * u.second) 

244 meas.notes["estimator"] = "utils.timer.timeMethod" 

245 if timings["StartTimestamp"]: 

246 meas.extras["start"] = Datum(timings["StartTimestamp"]) 

247 if timings["EndTimestamp"]: 

248 meas.extras["end"] = Datum(timings["EndTimestamp"]) 

249 return meas 

250 else: 

251 raise NoWorkFound(f"Nothing to do: no timing information for {self.config.target} found.") 

252 

253 

254# Expose MemoryMetricConfig name because config-writers expect it 

255MemoryMetricConfig = TimeMethodMetricConfig 

256 

257 

258class MemoryMetricTask(MetadataMetricTask): 

259 """A Task that computes the maximum resident set size using metadata 

260 produced by the `lsst.utils.timer.timeMethod` decorator. 

261 

262 Parameters 

263 ---------- 

264 args 

265 kwargs 

266 Constructor parameters are the same as for 

267 `lsst.verify.tasks.MetricTask`. 

268 """ 

269 

270 ConfigClass = MemoryMetricConfig 

271 _DefaultName = "memoryMetric" 

272 

273 @classmethod 

274 def getInputMetadataKeys(cls, config): 

275 """Get search strings for the metadata. 

276 

277 Parameters 

278 ---------- 

279 config : ``cls.ConfigClass`` 

280 Configuration for this task. 

281 

282 Returns 

283 ------- 

284 keys : `dict` 

285 A dictionary of keys, optionally prefixed by one or more tasks in 

286 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

287 

288 ``"EndMemory"`` 

289 The key for the memory usage at the end of the method (`str`). 

290 ``"MetadataVersion"`` 

291 The key for the task-level metadata version. 

292 """ 

293 keyBase = config.target 

294 # Parse keyBase to get just the task prefix, if any; needed to 

295 # guarantee that returned keys all point to unique entries. 

296 # The following line returns a "."-terminated string if keyBase has a 

297 # task prefix, and "" otherwise. 

298 taskPrefix = "".join(keyBase.rpartition(".")[0:2]) 

299 

300 return {"EndMemory": keyBase + "EndMaxResidentSetSize", 

301 "MetadataVersion": taskPrefix + "__version__", 

302 } 

303 

304 def makeMeasurement(self, memory): 

305 """Compute a maximum resident set size measurement from metadata 

306 provided by `lsst.utils.timer.timeMethod`. 

307 

308 Parameters 

309 ---------- 

310 memory : `dict` [`str`, any] 

311 A representation of the metadata passed to `run`. Each `dict` has 

312 the following keys: 

313 

314 ``"EndMemory"`` 

315 The memory usage at the end of the method (`int` or `None`). 

316 ``"MetadataVersion"`` 

317 The version of the task metadata in which the value was stored 

318 (`int` or `None`). `None` is assumed to be version 0. 

319 

320 Returns 

321 ------- 

322 measurement : `lsst.verify.Measurement` 

323 The maximum memory usage of the target method. 

324 

325 Raises 

326 ------ 

327 lsst.verify.tasks.MetricComputationError 

328 Raised if the memory metadata are invalid. 

329 lsst.pipe.base.NoWorkFound 

330 Raised if no matching memory metadata found. 

331 """ 

332 if memory["EndMemory"] is not None: 

333 try: 

334 maxMemory = int(memory["EndMemory"]) 

335 version = memory["MetadataVersion"] \ 

336 if memory["MetadataVersion"] else 0 

337 except (ValueError, TypeError) as e: 

338 raise MetricComputationError("Invalid metadata") from e 

339 else: 

340 meas = Measurement(self.config.metricName, 

341 self._addUnits(maxMemory, version)) 

342 meas.notes['estimator'] = 'utils.timer.timeMethod' 

343 return meas 

344 else: 

345 raise NoWorkFound(f"Nothing to do: no memory information for {self.config.target} found.") 

346 

347 def _addUnits(self, memory, version): 

348 """Represent memory usage in correct units. 

349 

350 Parameters 

351 ---------- 

352 memory : `int` 

353 The memory usage as returned by `resource.getrusage`, in 

354 platform-dependent units. 

355 version : `int` 

356 The metadata version. If ``0``, ``memory`` is in platform-dependent 

357 units. If ``1`` or greater, ``memory`` is in bytes. 

358 

359 Returns 

360 ------- 

361 memory : `astropy.units.Quantity` 

362 The memory usage in absolute units. 

363 """ 

364 if version >= 1: 

365 return memory * u.byte 

366 elif sys.platform.startswith('darwin'): 

367 # MacOS uses bytes 

368 return memory * u.byte 

369 elif sys.platform.startswith('sunos') \ 

370 or sys.platform.startswith('solaris'): 

371 # Solaris and SunOS use pages 

372 return memory * resource.getpagesize() * u.byte 

373 else: 

374 # Assume Linux, which uses kibibytes 

375 return memory * u.kibibyte