Coverage for python/lsst/verify/tasks/commonMetrics.py: 35%

77 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-08 14:45 -0800

1# 

2# This file is part of verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Code for measuring metrics that apply to any Task. 

25""" 

26 

27__all__ = ["TimingMetricConfig", "TimingMetricTask", 

28 "MemoryMetricConfig", "MemoryMetricTask", 

29 ] 

30 

31import resource 

32import sys 

33import warnings 

34 

35import astropy.units as u 

36 

37import lsst.pex.config as pexConfig 

38 

39from lsst.verify import Measurement, Datum 

40from lsst.verify.gen2tasks.metricRegistry import registerMultiple 

41from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \ 

42 MetadataMetricConfig 

43 

44 

45class TimeMethodMetricConfig(MetadataMetricConfig): 

46 """Common config fields for metrics based on 

47 `~lsst.utils.timer.timeMethod`. 

48 

49 These fields let metrics distinguish between different methods that have 

50 been decorated with `~lsst.utils.timer.timeMethod`. 

51 """ 

52 target = pexConfig.Field( 

53 dtype=str, 

54 doc="The method to profile, optionally prefixed by one or more tasks " 

55 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.") 

56 metric = pexConfig.Field( 

57 dtype=str, 

58 optional=True, 

59 doc="The fully qualified name of the metric to store the " 

60 "profiling information.", 

61 deprecated="This field has been replaced by connections.package and " 

62 "connections.metric. It will be removed along " 

63 "with daf_persistence." 

64 ) 

65 

66 def validate(self): 

67 super().validate() 

68 

69 if self.metric: 

70 if self.metric != self.connections.package \ 

71 + "." + self.connections.metric: 

72 warnings.warn( 

73 "config.metric is deprecated; set connections.package " 

74 "and connections.metric instead.", 

75 FutureWarning) 

76 try: 

77 self.connections.package, self.connections.metric \ 

78 = self.metric.split(".") 

79 except ValueError: 

80 self.connections.package = "" 

81 self.connections.metric = self.metric 

82 

83 

84# Expose TimingMetricConfig name because config-writers expect it 

85TimingMetricConfig = TimeMethodMetricConfig 

86 

87 

88@registerMultiple("timing") 

89class TimingMetricTask(MetadataMetricTask): 

90 """A Task that computes a wall-clock time using metadata produced by the 

91 `lsst.utils.timer.timeMethod` decorator. 

92 

93 Parameters 

94 ---------- 

95 args 

96 kwargs 

97 Constructor parameters are the same as for 

98 `lsst.verify.tasks.MetricTask`. 

99 """ 

100 

101 ConfigClass = TimingMetricConfig 

102 _DefaultName = "timingMetric" 

103 

104 @classmethod 

105 def getInputMetadataKeys(cls, config): 

106 """Get search strings for the metadata. 

107 

108 Parameters 

109 ---------- 

110 config : ``cls.ConfigClass`` 

111 Configuration for this task. 

112 

113 Returns 

114 ------- 

115 keys : `dict` 

116 A dictionary of keys, optionally prefixed by one or more tasks in 

117 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

118 

119 ``"StartTime"`` 

120 The key for when the target method started (`str`). 

121 ``"EndTime"`` 

122 The key for when the target method ended (`str`). 

123 ``"StartTimestamp"`` 

124 The key for an ISO 8601-compliant text string where the target 

125 method started (`str`). 

126 ``"EndTimestamp"`` 

127 The key for an ISO 8601-compliant text string where the target 

128 method ended (`str`). 

129 """ 

130 keyBase = config.target 

131 return {"StartTime": keyBase + "StartCpuTime", 

132 "EndTime": keyBase + "EndCpuTime", 

133 "StartTimestamp": keyBase + "StartUtc", 

134 "EndTimestamp": keyBase + "EndUtc", 

135 } 

136 

137 def makeMeasurement(self, timings): 

138 """Compute a wall-clock measurement from metadata provided by 

139 `lsst.utils.timer.timeMethod`. 

140 

141 Parameters 

142 ---------- 

143 timings : `dict` [`str`, any] 

144 A representation of the metadata passed to `run`. The `dict` has 

145 the following keys: 

146 

147 ``"StartTime"`` 

148 The time the target method started (`float` or `None`). 

149 ``"EndTime"`` 

150 The time the target method ended (`float` or `None`). 

151 ``"StartTimestamp"``, ``"EndTimestamp"`` 

152 The start and end timestamps, in an ISO 8601-compliant format 

153 (`str` or `None`). 

154 

155 Returns 

156 ------- 

157 measurement : `lsst.verify.Measurement` or `None` 

158 The running time of the target method. 

159 

160 Raises 

161 ------ 

162 MetricComputationError 

163 Raised if the timing metadata are invalid. 

164 """ 

165 if timings["StartTime"] is not None or timings["EndTime"] is not None: 

166 try: 

167 totalTime = timings["EndTime"] - timings["StartTime"] 

168 except TypeError: 

169 raise MetricComputationError("Invalid metadata") 

170 else: 

171 meas = Measurement(self.config.metricName, 

172 totalTime * u.second) 

173 meas.notes["estimator"] = "utils.timer.timeMethod" 

174 if timings["StartTimestamp"]: 

175 meas.extras["start"] = Datum(timings["StartTimestamp"]) 

176 if timings["EndTimestamp"]: 

177 meas.extras["end"] = Datum(timings["EndTimestamp"]) 

178 return meas 

179 else: 

180 self.log.info("Nothing to do: no timing information for %s found.", 

181 self.config.target) 

182 return None 

183 

184 

185# Expose MemoryMetricConfig name because config-writers expect it 

186MemoryMetricConfig = TimeMethodMetricConfig 

187 

188 

189@registerMultiple("memory") 

190class MemoryMetricTask(MetadataMetricTask): 

191 """A Task that computes the maximum resident set size using metadata 

192 produced by the `lsst.utils.timer.timeMethod` decorator. 

193 

194 Parameters 

195 ---------- 

196 args 

197 kwargs 

198 Constructor parameters are the same as for 

199 `lsst.verify.tasks.MetricTask`. 

200 """ 

201 

202 ConfigClass = MemoryMetricConfig 

203 _DefaultName = "memoryMetric" 

204 

205 @classmethod 

206 def getInputMetadataKeys(cls, config): 

207 """Get search strings for the metadata. 

208 

209 Parameters 

210 ---------- 

211 config : ``cls.ConfigClass`` 

212 Configuration for this task. 

213 

214 Returns 

215 ------- 

216 keys : `dict` 

217 A dictionary of keys, optionally prefixed by one or more tasks in 

218 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

219 

220 ``"EndMemory"`` 

221 The key for the memory usage at the end of the method (`str`). 

222 ``"MetadataVersion"`` 

223 The key for the task-level metadata version. 

224 """ 

225 keyBase = config.target 

226 # Parse keyBase to get just the task prefix, if any; needed to 

227 # guarantee that returned keys all point to unique entries. 

228 # The following line returns a "."-terminated string if keyBase has a 

229 # task prefix, and "" otherwise. 

230 taskPrefix = "".join(keyBase.rpartition(".")[0:2]) 

231 

232 return {"EndMemory": keyBase + "EndMaxResidentSetSize", 

233 "MetadataVersion": taskPrefix + "__version__", 

234 } 

235 

236 def makeMeasurement(self, memory): 

237 """Compute a maximum resident set size measurement from metadata 

238 provided by `lsst.utils.timer.timeMethod`. 

239 

240 Parameters 

241 ---------- 

242 memory : `dict` [`str`, any] 

243 A representation of the metadata passed to `run`. Each `dict` has 

244 the following keys: 

245 

246 ``"EndMemory"`` 

247 The memory usage at the end of the method (`int` or `None`). 

248 ``"MetadataVersion"`` 

249 The version of the task metadata in which the value was stored 

250 (`int` or `None`). `None` is assumed to be version 0. 

251 

252 Returns 

253 ------- 

254 measurement : `lsst.verify.Measurement` or `None` 

255 The maximum memory usage of the target method. 

256 

257 Raises 

258 ------ 

259 MetricComputationError 

260 Raised if the memory metadata are invalid. 

261 """ 

262 if memory["EndMemory"] is not None: 

263 try: 

264 maxMemory = int(memory["EndMemory"]) 

265 version = memory["MetadataVersion"] \ 

266 if memory["MetadataVersion"] else 0 

267 except (ValueError, TypeError) as e: 

268 raise MetricComputationError("Invalid metadata") from e 

269 else: 

270 meas = Measurement(self.config.metricName, 

271 self._addUnits(maxMemory, version)) 

272 meas.notes['estimator'] = 'utils.timer.timeMethod' 

273 return meas 

274 else: 

275 self.log.info("Nothing to do: no memory information for %s found.", 

276 self.config.target) 

277 return None 

278 

279 def _addUnits(self, memory, version): 

280 """Represent memory usage in correct units. 

281 

282 Parameters 

283 ---------- 

284 memory : `int` 

285 The memory usage as returned by `resource.getrusage`, in 

286 platform-dependent units. 

287 version : `int` 

288 The metadata version. If ``0``, ``memory`` is in platform-dependent 

289 units. If ``1`` or greater, ``memory`` is in bytes. 

290 

291 Returns 

292 ------- 

293 memory : `astropy.units.Quantity` 

294 The memory usage in absolute units. 

295 """ 

296 if version >= 1: 

297 return memory * u.byte 

298 elif sys.platform.startswith('darwin'): 

299 # MacOS uses bytes 

300 return memory * u.byte 

301 elif sys.platform.startswith('sunos') \ 

302 or sys.platform.startswith('solaris'): 

303 # Solaris and SunOS use pages 

304 return memory * resource.getpagesize() * u.byte 

305 else: 

306 # Assume Linux, which uses kibibytes 

307 return memory * u.kibibyte