Coverage for python/lsst/verify/tasks/commonMetrics.py: 37%

77 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-27 11:24 +0000

1# 

2# This file is part of verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Code for measuring metrics that apply to any Task. 

25""" 

26 

27__all__ = ["TimingMetricConfig", "TimingMetricTask", 

28 "MemoryMetricConfig", "MemoryMetricTask", 

29 ] 

30 

31import resource 

32import sys 

33import warnings 

34 

35import astropy.units as u 

36 

37import lsst.pex.config as pexConfig 

38 

39from lsst.verify import Measurement, Datum 

40from lsst.verify.gen2tasks.metricRegistry import registerMultiple 

41from lsst.verify.tasks import MetricComputationError, MetadataMetricTask, \ 

42 MetadataMetricConfig 

43 

44 

45class TimeMethodMetricConfig(MetadataMetricConfig): 

46 """Common config fields for metrics based on `~lsst.utils.timer.timeMethod`. 

47 

48 These fields let metrics distinguish between different methods that have 

49 been decorated with `~lsst.utils.timer.timeMethod`. 

50 """ 

51 target = pexConfig.Field( 

52 dtype=str, 

53 doc="The method to profile, optionally prefixed by one or more tasks " 

54 "in the format of `lsst.pipe.base.Task.getFullMetadata()`.") 

55 metric = pexConfig.Field( 

56 dtype=str, 

57 optional=True, 

58 doc="The fully qualified name of the metric to store the " 

59 "profiling information.", 

60 deprecated="This field has been replaced by connections.package and " 

61 "connections.metric. It will be removed along " 

62 "with daf_persistence." 

63 ) 

64 

65 def validate(self): 

66 super().validate() 

67 

68 if self.metric: 

69 if self.metric != self.connections.package \ 

70 + "." + self.connections.metric: 

71 warnings.warn( 

72 "config.metric is deprecated; set connections.package " 

73 "and connections.metric instead.", 

74 FutureWarning) 

75 try: 

76 self.connections.package, self.connections.metric \ 

77 = self.metric.split(".") 

78 except ValueError: 

79 self.connections.package = "" 

80 self.connections.metric = self.metric 

81 

82 

83# Expose TimingMetricConfig name because config-writers expect it 

84TimingMetricConfig = TimeMethodMetricConfig 

85 

86 

87@registerMultiple("timing") 

88class TimingMetricTask(MetadataMetricTask): 

89 """A Task that computes a wall-clock time using metadata produced by the 

90 `lsst.utils.timer.timeMethod` decorator. 

91 

92 Parameters 

93 ---------- 

94 args 

95 kwargs 

96 Constructor parameters are the same as for 

97 `lsst.verify.tasks.MetricTask`. 

98 """ 

99 

100 ConfigClass = TimingMetricConfig 

101 _DefaultName = "timingMetric" 

102 

103 @classmethod 

104 def getInputMetadataKeys(cls, config): 

105 """Get search strings for the metadata. 

106 

107 Parameters 

108 ---------- 

109 config : ``cls.ConfigClass`` 

110 Configuration for this task. 

111 

112 Returns 

113 ------- 

114 keys : `dict` 

115 A dictionary of keys, optionally prefixed by one or more tasks in 

116 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

117 

118 ``"StartTime"`` 

119 The key for when the target method started (`str`). 

120 ``"EndTime"`` 

121 The key for when the target method ended (`str`). 

122 ``"StartTimestamp"`` 

123 The key for an ISO 8601-compliant text string where the target 

124 method started (`str`). 

125 ``"EndTimestamp"`` 

126 The key for an ISO 8601-compliant text string where the target 

127 method ended (`str`). 

128 """ 

129 keyBase = config.target 

130 return {"StartTime": keyBase + "StartCpuTime", 

131 "EndTime": keyBase + "EndCpuTime", 

132 "StartTimestamp": keyBase + "StartUtc", 

133 "EndTimestamp": keyBase + "EndUtc", 

134 } 

135 

136 def makeMeasurement(self, timings): 

137 """Compute a wall-clock measurement from metadata provided by 

138 `lsst.utils.timer.timeMethod`. 

139 

140 Parameters 

141 ---------- 

142 timings : `dict` [`str`, any] 

143 A representation of the metadata passed to `run`. The `dict` has 

144 the following keys: 

145 

146 ``"StartTime"`` 

147 The time the target method started (`float` or `None`). 

148 ``"EndTime"`` 

149 The time the target method ended (`float` or `None`). 

150 ``"StartTimestamp"``, ``"EndTimestamp"`` 

151 The start and end timestamps, in an ISO 8601-compliant format 

152 (`str` or `None`). 

153 

154 Returns 

155 ------- 

156 measurement : `lsst.verify.Measurement` or `None` 

157 The running time of the target method. 

158 

159 Raises 

160 ------ 

161 MetricComputationError 

162 Raised if the timing metadata are invalid. 

163 """ 

164 if timings["StartTime"] is not None or timings["EndTime"] is not None: 

165 try: 

166 totalTime = timings["EndTime"] - timings["StartTime"] 

167 except TypeError: 

168 raise MetricComputationError("Invalid metadata") 

169 else: 

170 meas = Measurement(self.config.metricName, 

171 totalTime * u.second) 

172 meas.notes["estimator"] = "utils.timer.timeMethod" 

173 if timings["StartTimestamp"]: 

174 meas.extras["start"] = Datum(timings["StartTimestamp"]) 

175 if timings["EndTimestamp"]: 

176 meas.extras["end"] = Datum(timings["EndTimestamp"]) 

177 return meas 

178 else: 

179 self.log.info("Nothing to do: no timing information for %s found.", 

180 self.config.target) 

181 return None 

182 

183 

184# Expose MemoryMetricConfig name because config-writers expect it 

185MemoryMetricConfig = TimeMethodMetricConfig 

186 

187 

188@registerMultiple("memory") 

189class MemoryMetricTask(MetadataMetricTask): 

190 """A Task that computes the maximum resident set size using metadata 

191 produced by the `lsst.utils.timer.timeMethod` decorator. 

192 

193 Parameters 

194 ---------- 

195 args 

196 kwargs 

197 Constructor parameters are the same as for 

198 `lsst.verify.tasks.MetricTask`. 

199 """ 

200 

201 ConfigClass = MemoryMetricConfig 

202 _DefaultName = "memoryMetric" 

203 

204 @classmethod 

205 def getInputMetadataKeys(cls, config): 

206 """Get search strings for the metadata. 

207 

208 Parameters 

209 ---------- 

210 config : ``cls.ConfigClass`` 

211 Configuration for this task. 

212 

213 Returns 

214 ------- 

215 keys : `dict` 

216 A dictionary of keys, optionally prefixed by one or more tasks in 

217 the format of `lsst.pipe.base.Task.getFullMetadata()`. 

218 

219 ``"EndMemory"`` 

220 The key for the memory usage at the end of the method (`str`). 

221 ``"MetadataVersion"`` 

222 The key for the task-level metadata version. 

223 """ 

224 keyBase = config.target 

225 # Parse keyBase to get just the task prefix, if any; needed to 

226 # guarantee that returned keys all point to unique entries. 

227 # The following line returns a "."-terminated string if keyBase has a 

228 # task prefix, and "" otherwise. 

229 taskPrefix = "".join(keyBase.rpartition(".")[0:2]) 

230 

231 return {"EndMemory": keyBase + "EndMaxResidentSetSize", 

232 "MetadataVersion": taskPrefix + "__version__", 

233 } 

234 

235 def makeMeasurement(self, memory): 

236 """Compute a maximum resident set size measurement from metadata 

237 provided by `lsst.utils.timer.timeMethod`. 

238 

239 Parameters 

240 ---------- 

241 memory : `dict` [`str`, any] 

242 A representation of the metadata passed to `run`. Each `dict` has 

243 the following keys: 

244 

245 ``"EndMemory"`` 

246 The memory usage at the end of the method (`int` or `None`). 

247 ``"MetadataVersion"`` 

248 The version of the task metadata in which the value was stored 

249 (`int` or `None`). `None` is assumed to be version 0. 

250 

251 Returns 

252 ------- 

253 measurement : `lsst.verify.Measurement` or `None` 

254 The maximum memory usage of the target method. 

255 

256 Raises 

257 ------ 

258 MetricComputationError 

259 Raised if the memory metadata are invalid. 

260 """ 

261 if memory["EndMemory"] is not None: 

262 try: 

263 maxMemory = int(memory["EndMemory"]) 

264 version = memory["MetadataVersion"] \ 

265 if memory["MetadataVersion"] else 0 

266 except (ValueError, TypeError) as e: 

267 raise MetricComputationError("Invalid metadata") from e 

268 else: 

269 meas = Measurement(self.config.metricName, 

270 self._addUnits(maxMemory, version)) 

271 meas.notes['estimator'] = 'utils.timer.timeMethod' 

272 return meas 

273 else: 

274 self.log.info("Nothing to do: no memory information for %s found.", 

275 self.config.target) 

276 return None 

277 

278 def _addUnits(self, memory, version): 

279 """Represent memory usage in correct units. 

280 

281 Parameters 

282 ---------- 

283 memory : `int` 

284 The memory usage as returned by `resource.getrusage`, in 

285 platform-dependent units. 

286 version : `int` 

287 The metadata version. If ``0``, ``memory`` is in platform-dependent 

288 units. If ``1`` or greater, ``memory`` is in bytes. 

289 

290 Returns 

291 ------- 

292 memory : `astropy.units.Quantity` 

293 The memory usage in absolute units. 

294 """ 

295 if version >= 1: 

296 return memory * u.byte 

297 elif sys.platform.startswith('darwin'): 

298 # MacOS uses bytes 

299 return memory * u.byte 

300 elif sys.platform.startswith('sunos') \ 

301 or sys.platform.startswith('solaris'): 

302 # Solaris and SunOS use pages 

303 return memory * resource.getpagesize() * u.byte 

304 else: 

305 # Assume Linux, which uses kibibytes 

306 return memory * u.kibibyte