Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11# 

12 

13"""Utilities for measuring execution time. 

14""" 

15 

16from __future__ import annotations 

17 

18__all__ = ["logInfo", "timeMethod", "time_this"] 

19 

20import functools 

21import logging 

22import resource 

23import time 

24import datetime 

25import traceback 

26from contextlib import contextmanager 

27 

28from typing import ( 

29 Any, 

30 Callable, 

31 Collection, 

32 Iterable, 

33 Iterator, 

34 MutableMapping, 

35 Optional, 

36 Tuple, 

37 TYPE_CHECKING, 

38) 

39 

40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 from .logging import LsstLoggers 

42 

43 

44def _add_to_metadata(metadata: MutableMapping, name: str, value: Any) -> None: 

45 """Add a value to dict-like object, creating list as needed. 

46 

47 The list grows as more values are added for that key. 

48 

49 Parameters 

50 ---------- 

51 metadata : `dict`-like, optional 

52 `dict`-like object that can store keys. Uses `add()` method if 

53 one is available, else creates list and appends value if needed. 

54 name : `str` 

55 The key to use in the metadata dictionary. 

56 value : Any 

57 Value to store in the list. 

58 """ 

59 try: 

60 try: 

61 # PropertySet should always prefer LongLong for integers 

62 metadata.addLongLong(name, value) # type: ignore 

63 except TypeError: 

64 metadata.add(name, value) # type: ignore 

65 except AttributeError: 

66 pass 

67 else: 

68 return 

69 

70 # Fallback code where `add` is not implemented. 

71 if name not in metadata: 

72 metadata[name] = [] 

73 metadata[name].append(value) 

74 

75 

76def _find_outside_stacklevel() -> int: 

77 """Find the stack level corresponding to caller code outside of this 

78 module. 

79 

80 This can be passed directly to `logging.Logger.log()` to ensure 

81 that log messages are issued as if they are coming from caller code. 

82 

83 Returns 

84 ------- 

85 stacklevel : `int` 

86 The stack level to use to refer to a caller outside of this module. 

87 A ``stacklevel`` of ``1`` corresponds to the caller of this internal 

88 function and that is the default expected by `logging.Logger.log()`. 

89 

90 Notes 

91 ----- 

92 Intended to be called from the function that is going to issue a log 

93 message. The result should be passed into `~logging.Logger.log` via the 

94 keyword parameter ``stacklevel``. 

95 """ 

96 stacklevel = 1 # the default for `Logger.log` 

97 stack = traceback.extract_stack() 

98 for i, s in enumerate(reversed(stack)): 

99 if "lsst/utils" not in s.filename: 

100 # 0 will be this function. 

101 # 1 will be the caller which will be the default for `Logger.log` 

102 # and so does not need adjustment. 

103 stacklevel = i 

104 break 

105 

106 return stacklevel 

107 

108 

109def logPairs(obj: Any, pairs: Collection[Tuple[str, Any]], logLevel: int = logging.DEBUG, 

110 metadata: Optional[MutableMapping] = None, 

111 logger: Optional[logging.Logger] = None) -> None: 

112 """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log`` 

113 

114 Parameters 

115 ---------- 

116 obj : `object` 

117 An object with one or both of these two attributes: 

118 

119 - ``metadata`` a `dict`-like container for storing metadata. 

120 Can use the ``add(name, value)`` method if found, else will append 

121 entries to a list. 

122 - ``log`` an instance of `logging.Logger` or subclass. 

123 

124 If `None`, at least one of ``metadata`` or ``logger`` should be passed 

125 or this function will do nothing. 

126 pairs : sequence 

127 A sequence of ``(name, value)`` pairs, with value typically numeric. 

128 logLevel : `int, optional 

129 Log level (an `logging` level constant, such as `logging.DEBUG`). 

130 metadata : `collections.abc.MutableMapping`, optional 

131 Metadata object to write entries to. Ignored if `None`. 

132 logger : `logging.Logger` 

133 Log object to write entries to. Ignored if `None`. 

134 """ 

135 if obj is not None: 

136 if metadata is None: 

137 try: 

138 metadata = obj.metadata 

139 except AttributeError: 

140 pass 

141 if logger is None: 

142 try: 

143 logger = obj.log 

144 except AttributeError: 

145 pass 

146 strList = [] 

147 for name, value in pairs: 

148 if metadata is not None: 

149 _add_to_metadata(metadata, name, value) 

150 strList.append(f"{name}={value}") 

151 if logger is not None: 

152 # Want the file associated with this log message to be that 

153 # of the caller. 

154 stacklevel = _find_outside_stacklevel() 

155 logging.getLogger("timer." + logger.name).log(logLevel, "; ".join(strList), stacklevel=stacklevel) 

156 

157 

158def logInfo(obj: Any, prefix: str, logLevel: int = logging.DEBUG, 

159 metadata: Optional[MutableMapping] = None, logger: Optional[logging.Logger] = None) -> None: 

160 """Log timer information to ``obj.metadata`` and ``obj.log``. 

161 

162 Parameters 

163 ---------- 

164 obj : `object` 

165 An object with both or one these two attributes: 

166 

167 - ``metadata`` a `dict`-like container for storing metadata. 

168 Can use the ``add(name, value)`` method if found, else will append 

169 entries to a list. 

170 - ``log`` an instance of `logging.Logger` or subclass. 

171 

172 If `None`, at least one of ``metadata`` or ``logger`` should be passed 

173 or this function will do nothing. 

174 prefix : `str` 

175 Name prefix, the resulting entries are ``CpuTime``, etc.. For example 

176 `timeMethod` uses ``prefix = Start`` when the method begins and 

177 ``prefix = End`` when the method ends. 

178 logLevel : optional 

179 Log level (a `logging` level constant, such as `logging.DEBUG`). 

180 metadata : `collections.abc.MutableMapping`, optional 

181 Metadata object to write entries to, overriding ``obj.metadata``. 

182 logger : `logging.Logger` 

183 Log object to write entries to, overriding ``obj.log``. 

184 

185 Notes 

186 ----- 

187 Logged items include: 

188 

189 - ``Utc``: UTC date in ISO format (only in metadata since log entries have 

190 timestamps). 

191 - ``CpuTime``: System + User CPU time (seconds). This should only be used 

192 in differential measurements; the time reference point is undefined. 

193 - ``MaxRss``: maximum resident set size. 

194 

195 All logged resource information is only for the current process; child 

196 processes are excluded. 

197 """ 

198 cpuTime = time.process_time() 

199 res = resource.getrusage(resource.RUSAGE_SELF) 

200 if metadata is None and obj is not None: 

201 try: 

202 metadata = obj.metadata 

203 except AttributeError: 

204 pass 

205 if metadata is not None: 

206 # Log messages already have timestamps. 

207 utcStr = datetime.datetime.utcnow().isoformat() 

208 _add_to_metadata(metadata, name=prefix + "Utc", value=utcStr) 

209 logPairs(obj=obj, 

210 pairs=[ 

211 (prefix + "CpuTime", cpuTime), 

212 (prefix + "UserTime", res.ru_utime), 

213 (prefix + "SystemTime", res.ru_stime), 

214 (prefix + "MaxResidentSetSize", int(res.ru_maxrss)), 

215 (prefix + "MinorPageFaults", int(res.ru_minflt)), 

216 (prefix + "MajorPageFaults", int(res.ru_majflt)), 

217 (prefix + "BlockInputs", int(res.ru_inblock)), 

218 (prefix + "BlockOutputs", int(res.ru_oublock)), 

219 (prefix + "VoluntaryContextSwitches", int(res.ru_nvcsw)), 

220 (prefix + "InvoluntaryContextSwitches", int(res.ru_nivcsw)), 

221 ], 

222 logLevel=logLevel, 

223 metadata=metadata, 

224 logger=logger) 

225 

226 

227def timeMethod(_func: Optional[Any] = None, *, metadata: Optional[MutableMapping] = None, 

228 logger: Optional[logging.Logger] = None, 

229 logLevel: int = logging.DEBUG) -> Callable: 

230 """Decorator to measure duration of a method. 

231 

232 Parameters 

233 ---------- 

234 func 

235 The method to wrap. 

236 metadata : `collections.abc.MutableMapping`, optional 

237 Metadata to use as override if the instance object attached 

238 to this timer does not support a ``metadata`` property. 

239 logger : `logging.Logger`, optional 

240 Logger to use when the class containing the decorated method does not 

241 have a ``log`` property. 

242 logLevel : `int`, optional 

243 Log level to use when logging messages. Default is `~logging.DEBUG`. 

244 

245 Notes 

246 ----- 

247 Writes various measures of time and possibly memory usage to the 

248 metadata; all items are prefixed with the function name. 

249 

250 .. warning:: 

251 

252 This decorator only works with instance methods of any class 

253 with these attributes: 

254 

255 - ``metadata``: an instance of `collections.abc.Mapping`. The ``add`` 

256 method will be used if available, else entries will be added to a 

257 list. 

258 - ``log``: an instance of `logging.Logger` or subclass. 

259 

260 Examples 

261 -------- 

262 To use: 

263 

264 .. code-block:: python 

265 

266 import lsst.utils as utils 

267 import lsst.pipe.base as pipeBase 

268 class FooTask(pipeBase.Task): 

269 pass 

270 

271 @utils.timeMethod 

272 def run(self, ...): # or any other instance method you want to time 

273 pass 

274 """ 

275 

276 def decorator_timer(func: Callable) -> Callable: 

277 @functools.wraps(func) 

278 def wrapper(self: Any, *args: Any, **keyArgs: Any) -> Any: 

279 logInfo(obj=self, prefix=func.__name__ + "Start", metadata=metadata, logger=logger, 

280 logLevel=logLevel) 

281 try: 

282 res = func(self, *args, **keyArgs) 

283 finally: 

284 logInfo(obj=self, prefix=func.__name__ + "End", metadata=metadata, logger=logger, 

285 logLevel=logLevel) 

286 return res 

287 return wrapper 

288 

289 if _func is None: 

290 return decorator_timer 

291 else: 

292 return decorator_timer(_func) 

293 

294 

295@contextmanager 

296def time_this(log: Optional[LsstLoggers] = None, msg: Optional[str] = None, 

297 level: int = logging.DEBUG, prefix: Optional[str] = "timer", 

298 args: Iterable[Any] = ()) -> Iterator[None]: 

299 """Time the enclosed block and issue a log message. 

300 

301 Parameters 

302 ---------- 

303 log : `logging.Logger`, optional 

304 Logger to use to report the timer message. The root logger will 

305 be used if none is given. 

306 msg : `str`, optional 

307 Context to include in log message. 

308 level : `int`, optional 

309 Python logging level to use to issue the log message. If the 

310 code block raises an exception the log message will automatically 

311 switch to level ERROR. 

312 prefix : `str`, optional 

313 Prefix to use to prepend to the supplied logger to 

314 create a new logger to use instead. No prefix is used if the value 

315 is set to `None`. Defaults to "timer". 

316 args : iterable of any 

317 Additional parameters passed to the log command that should be 

318 written to ``msg``. 

319 """ 

320 if log is None: 

321 log = logging.getLogger() 

322 if prefix: 

323 log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix 

324 log = logging.getLogger(log_name) 

325 

326 success = False 

327 start = time.time() 

328 try: 

329 yield 

330 success = True 

331 finally: 

332 end = time.time() 

333 

334 # The message is pre-inserted to allow the logger to expand 

335 # the additional args provided. Make that easier by converting 

336 # the None message to empty string. 

337 if msg is None: 

338 msg = "" 

339 

340 if not success: 

341 # Something went wrong so change the log level to indicate 

342 # this. 

343 level = logging.ERROR 

344 

345 # Specify stacklevel to ensure the message is reported from the 

346 # caller (1 is this file, 2 is contextlib, 3 is user) 

347 log.log(level, msg + "%sTook %.4f seconds", *args, 

348 ": " if msg else "", end - start, stacklevel=3)