Coverage for python/lsst/utils/timer.py: 19%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11#
13"""Utilities for measuring execution time.
14"""
16from __future__ import annotations
18__all__ = ["logInfo", "timeMethod", "time_this"]
20import datetime
21import functools
22import inspect
23import logging
24import resource
25import time
26from contextlib import contextmanager
27from typing import (
28 TYPE_CHECKING,
29 Any,
30 Callable,
31 Collection,
32 Iterable,
33 Iterator,
34 MutableMapping,
35 Optional,
36 Tuple,
37)
39if TYPE_CHECKING: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true
40 from .logging import LsstLoggers
43def _add_to_metadata(metadata: MutableMapping, name: str, value: Any) -> None:
44 """Add a value to dict-like object, creating list as needed.
46 The list grows as more values are added for that key.
48 Parameters
49 ----------
50 metadata : `dict`-like, optional
51 `dict`-like object that can store keys. Uses `add()` method if
52 one is available, else creates list and appends value if needed.
53 name : `str`
54 The key to use in the metadata dictionary.
55 value : Any
56 Value to store in the list.
57 """
58 try:
59 try:
60 # PropertySet should always prefer LongLong for integers
61 metadata.addLongLong(name, value) # type: ignore
62 except (TypeError, AttributeError):
63 metadata.add(name, value) # type: ignore
64 except AttributeError:
65 pass
66 else:
67 return
69 # Fallback code where `add` is not implemented.
70 if name not in metadata:
71 metadata[name] = []
72 metadata[name].append(value)
75def _find_outside_stacklevel() -> int:
76 """Find the stack level corresponding to caller code outside of this
77 module.
79 This can be passed directly to `logging.Logger.log()` to ensure
80 that log messages are issued as if they are coming from caller code.
82 Returns
83 -------
84 stacklevel : `int`
85 The stack level to use to refer to a caller outside of this module.
86 A ``stacklevel`` of ``1`` corresponds to the caller of this internal
87 function and that is the default expected by `logging.Logger.log()`.
89 Notes
90 -----
91 Intended to be called from the function that is going to issue a log
92 message. The result should be passed into `~logging.Logger.log` via the
93 keyword parameter ``stacklevel``.
94 """
95 stacklevel = 1 # the default for `Logger.log`
96 for i, s in enumerate(inspect.stack()):
97 module = inspect.getmodule(s.frame)
98 if module is None:
99 # Stack frames sometimes hang around so explicilty delete.
100 del s
101 continue
102 if not module.__name__.startswith("lsst.utils"):
103 # 0 will be this function.
104 # 1 will be the caller which will be the default for `Logger.log`
105 # and so does not need adjustment.
106 stacklevel = i
107 break
108 # Stack frames sometimes hang around so explicilty delete.
109 del s
111 return stacklevel
114def logPairs(
115 obj: Any,
116 pairs: Collection[Tuple[str, Any]],
117 logLevel: int = logging.DEBUG,
118 metadata: Optional[MutableMapping] = None,
119 logger: Optional[logging.Logger] = None,
120 stacklevel: Optional[int] = None,
121) -> None:
122 """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``
124 Parameters
125 ----------
126 obj : `object`
127 An object with one or both of these two attributes:
129 - ``metadata`` a `dict`-like container for storing metadata.
130 Can use the ``add(name, value)`` method if found, else will append
131 entries to a list.
132 - ``log`` an instance of `logging.Logger` or subclass.
134 If `None`, at least one of ``metadata`` or ``logger`` should be passed
135 or this function will do nothing.
136 pairs : sequence
137 A sequence of ``(name, value)`` pairs, with value typically numeric.
138 logLevel : `int, optional
139 Log level (an `logging` level constant, such as `logging.DEBUG`).
140 metadata : `collections.abc.MutableMapping`, optional
141 Metadata object to write entries to. Ignored if `None`.
142 logger : `logging.Logger`
143 Log object to write entries to. Ignored if `None`.
144 stacklevel : `int`, optional
145 The stack level to pass to the logger to adjust which stack frame
146 is used to report the file information. If `None` the stack level
147 is computed such that it is reported as the first package outside
148 of the utils package. If a value is given here it is adjusted by
149 1 to account for this caller.
150 """
151 if obj is not None:
152 if metadata is None:
153 try:
154 metadata = obj.metadata
155 except AttributeError:
156 pass
157 if logger is None:
158 try:
159 logger = obj.log
160 except AttributeError:
161 pass
162 strList = []
163 for name, value in pairs:
164 if metadata is not None:
165 _add_to_metadata(metadata, name, value)
166 strList.append(f"{name}={value}")
167 if logger is not None:
168 # Want the file associated with this log message to be that
169 # of the caller. This is expensive so only do it if we know the
170 # message will be issued.
171 timer_logger = logging.getLogger("timer." + logger.name)
172 if timer_logger.isEnabledFor(logLevel):
173 if stacklevel is None:
174 stacklevel = _find_outside_stacklevel()
175 else:
176 # Account for the caller stack.
177 stacklevel += 1
178 timer_logger.log(logLevel, "; ".join(strList), stacklevel=stacklevel)
181def logInfo(
182 obj: Any,
183 prefix: str,
184 logLevel: int = logging.DEBUG,
185 metadata: Optional[MutableMapping] = None,
186 logger: Optional[logging.Logger] = None,
187 stacklevel: Optional[int] = None,
188) -> None:
189 """Log timer information to ``obj.metadata`` and ``obj.log``.
191 Parameters
192 ----------
193 obj : `object`
194 An object with both or one these two attributes:
196 - ``metadata`` a `dict`-like container for storing metadata.
197 Can use the ``add(name, value)`` method if found, else will append
198 entries to a list.
199 - ``log`` an instance of `logging.Logger` or subclass.
201 If `None`, at least one of ``metadata`` or ``logger`` should be passed
202 or this function will do nothing.
203 prefix : `str`
204 Name prefix, the resulting entries are ``CpuTime``, etc.. For example
205 `timeMethod` uses ``prefix = Start`` when the method begins and
206 ``prefix = End`` when the method ends.
207 logLevel : optional
208 Log level (a `logging` level constant, such as `logging.DEBUG`).
209 metadata : `collections.abc.MutableMapping`, optional
210 Metadata object to write entries to, overriding ``obj.metadata``.
211 logger : `logging.Logger`
212 Log object to write entries to, overriding ``obj.log``.
213 stacklevel : `int`, optional
214 The stack level to pass to the logger to adjust which stack frame
215 is used to report the file information. If `None` the stack level
216 is computed such that it is reported as the first package outside
217 of the utils package. If a value is given here it is adjusted by
218 1 to account for this caller.
220 Notes
221 -----
222 Logged items include:
224 - ``Utc``: UTC date in ISO format (only in metadata since log entries have
225 timestamps).
226 - ``CpuTime``: System + User CPU time (seconds). This should only be used
227 in differential measurements; the time reference point is undefined.
228 - ``MaxRss``: maximum resident set size.
230 All logged resource information is only for the current process; child
231 processes are excluded.
232 """
233 cpuTime = time.process_time()
234 res = resource.getrusage(resource.RUSAGE_SELF)
235 if metadata is None and obj is not None:
236 try:
237 metadata = obj.metadata
238 except AttributeError:
239 pass
240 if metadata is not None:
241 # Log messages already have timestamps.
242 utcStr = datetime.datetime.utcnow().isoformat()
243 _add_to_metadata(metadata, name=prefix + "Utc", value=utcStr)
244 if stacklevel is not None:
245 # Account for the caller of this routine not knowing that we
246 # are going one down in the stack.
247 stacklevel += 1
248 logPairs(
249 obj=obj,
250 pairs=[
251 (prefix + "CpuTime", cpuTime),
252 (prefix + "UserTime", res.ru_utime),
253 (prefix + "SystemTime", res.ru_stime),
254 (prefix + "MaxResidentSetSize", int(res.ru_maxrss)),
255 (prefix + "MinorPageFaults", int(res.ru_minflt)),
256 (prefix + "MajorPageFaults", int(res.ru_majflt)),
257 (prefix + "BlockInputs", int(res.ru_inblock)),
258 (prefix + "BlockOutputs", int(res.ru_oublock)),
259 (prefix + "VoluntaryContextSwitches", int(res.ru_nvcsw)),
260 (prefix + "InvoluntaryContextSwitches", int(res.ru_nivcsw)),
261 ],
262 logLevel=logLevel,
263 metadata=metadata,
264 logger=logger,
265 stacklevel=stacklevel,
266 )
269def timeMethod(
270 _func: Optional[Any] = None,
271 *,
272 metadata: Optional[MutableMapping] = None,
273 logger: Optional[logging.Logger] = None,
274 logLevel: int = logging.DEBUG,
275) -> Callable:
276 """Measure duration of a method.
278 Parameters
279 ----------
280 func
281 The method to wrap.
282 metadata : `collections.abc.MutableMapping`, optional
283 Metadata to use as override if the instance object attached
284 to this timer does not support a ``metadata`` property.
285 logger : `logging.Logger`, optional
286 Logger to use when the class containing the decorated method does not
287 have a ``log`` property.
288 logLevel : `int`, optional
289 Log level to use when logging messages. Default is `~logging.DEBUG`.
291 Notes
292 -----
293 Writes various measures of time and possibly memory usage to the
294 metadata; all items are prefixed with the function name.
296 .. warning::
298 This decorator only works with instance methods of any class
299 with these attributes:
301 - ``metadata``: an instance of `collections.abc.Mapping`. The ``add``
302 method will be used if available, else entries will be added to a
303 list.
304 - ``log``: an instance of `logging.Logger` or subclass.
306 Examples
307 --------
308 To use:
310 .. code-block:: python
312 import lsst.utils as utils
313 import lsst.pipe.base as pipeBase
314 class FooTask(pipeBase.Task):
315 pass
317 @utils.timeMethod
318 def run(self, ...): # or any other instance method you want to time
319 pass
320 """
322 def decorator_timer(func: Callable) -> Callable:
323 @functools.wraps(func)
324 def timeMethod_wrapper(self: Any, *args: Any, **keyArgs: Any) -> Any:
325 # Adjust the stacklevel to account for the wrappers.
326 # stacklevel 1 would make the log message come from this function
327 # but we want it to come from the file that defined the method
328 # so need to increment by 1 to get to the caller.
329 stacklevel = 2
330 logInfo(
331 obj=self,
332 prefix=func.__name__ + "Start",
333 metadata=metadata,
334 logger=logger,
335 logLevel=logLevel,
336 stacklevel=stacklevel,
337 )
338 try:
339 res = func(self, *args, **keyArgs)
340 finally:
341 logInfo(
342 obj=self,
343 prefix=func.__name__ + "End",
344 metadata=metadata,
345 logger=logger,
346 logLevel=logLevel,
347 stacklevel=stacklevel,
348 )
349 return res
351 return timeMethod_wrapper
353 if _func is None:
354 return decorator_timer
355 else:
356 return decorator_timer(_func)
359@contextmanager
360def time_this(
361 log: Optional[LsstLoggers] = None,
362 msg: Optional[str] = None,
363 level: int = logging.DEBUG,
364 prefix: Optional[str] = "timer",
365 args: Iterable[Any] = (),
366) -> Iterator[None]:
367 """Time the enclosed block and issue a log message.
369 Parameters
370 ----------
371 log : `logging.Logger`, optional
372 Logger to use to report the timer message. The root logger will
373 be used if none is given.
374 msg : `str`, optional
375 Context to include in log message.
376 level : `int`, optional
377 Python logging level to use to issue the log message. If the
378 code block raises an exception the log message will automatically
379 switch to level ERROR.
380 prefix : `str`, optional
381 Prefix to use to prepend to the supplied logger to
382 create a new logger to use instead. No prefix is used if the value
383 is set to `None`. Defaults to "timer".
384 args : iterable of any
385 Additional parameters passed to the log command that should be
386 written to ``msg``.
387 """
388 if log is None:
389 log = logging.getLogger()
390 if prefix:
391 log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix
392 log = logging.getLogger(log_name)
394 success = False
395 start = time.time()
396 try:
397 yield
398 success = True
399 finally:
400 end = time.time()
402 # The message is pre-inserted to allow the logger to expand
403 # the additional args provided. Make that easier by converting
404 # the None message to empty string.
405 if msg is None:
406 msg = ""
408 if not success:
409 # Something went wrong so change the log level to indicate
410 # this.
411 level = logging.ERROR
413 # Specify stacklevel to ensure the message is reported from the
414 # caller (1 is this file, 2 is contextlib, 3 is user)
415 log.log(level, msg + "%sTook %.4f seconds", *args, ": " if msg else "", end - start, stacklevel=3)