Coverage for python/lsst/utils/timer.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11#
13"""Utilities for measuring execution time.
14"""
16from __future__ import annotations
18__all__ = ["logInfo", "timeMethod", "time_this"]
20import functools
21import logging
22import resource
23import time
24import datetime
25import traceback
26from contextlib import contextmanager
28from typing import (
29 Any,
30 Callable,
31 Collection,
32 Iterable,
33 Iterator,
34 MutableMapping,
35 Optional,
36 Tuple,
37 TYPE_CHECKING,
38)
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from .logging import LsstLoggers
44def _add_to_metadata(metadata: MutableMapping, name: str, value: Any) -> None:
45 """Add a value to dict-like object, creating list as needed.
47 The list grows as more values are added for that key.
49 Parameters
50 ----------
51 metadata : `dict`-like, optional
52 `dict`-like object that can store keys. Uses `add()` method if
53 one is available, else creates list and appends value if needed.
54 name : `str`
55 The key to use in the metadata dictionary.
56 value : Any
57 Value to store in the list.
58 """
59 try:
60 try:
61 # PropertySet should always prefer LongLong for integers
62 metadata.addLongLong(name, value) # type: ignore
63 except TypeError:
64 metadata.add(name, value) # type: ignore
65 except AttributeError:
66 pass
67 else:
68 return
70 # Fallback code where `add` is not implemented.
71 if name not in metadata:
72 metadata[name] = []
73 metadata[name].append(value)
76def _find_outside_stacklevel() -> int:
77 """Find the stack level corresponding to caller code outside of this
78 module.
80 This can be passed directly to `logging.Logger.log()` to ensure
81 that log messages are issued as if they are coming from caller code.
83 Returns
84 -------
85 stacklevel : `int`
86 The stack level to use to refer to a caller outside of this module.
87 A ``stacklevel`` of ``1`` corresponds to the caller of this internal
88 function and that is the default expected by `logging.Logger.log()`.
90 Notes
91 -----
92 Intended to be called from the function that is going to issue a log
93 message. The result should be passed into `~logging.Logger.log` via the
94 keyword parameter ``stacklevel``.
95 """
96 stacklevel = 1 # the default for `Logger.log`
97 stack = traceback.extract_stack()
98 for i, s in enumerate(reversed(stack)):
99 if "lsst/utils" not in s.filename:
100 # 0 will be this function.
101 # 1 will be the caller which will be the default for `Logger.log`
102 # and so does not need adjustment.
103 stacklevel = i
104 break
106 return stacklevel
109def logPairs(obj: Any, pairs: Collection[Tuple[str, Any]], logLevel: int = logging.DEBUG,
110 metadata: Optional[MutableMapping] = None,
111 logger: Optional[logging.Logger] = None) -> None:
112 """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``
114 Parameters
115 ----------
116 obj : `object`
117 An object with one or both of these two attributes:
119 - ``metadata`` a `dict`-like container for storing metadata.
120 Can use the ``add(name, value)`` method if found, else will append
121 entries to a list.
122 - ``log`` an instance of `logging.Logger` or subclass.
124 If `None`, at least one of ``metadata`` or ``logger`` should be passed
125 or this function will do nothing.
126 pairs : sequence
127 A sequence of ``(name, value)`` pairs, with value typically numeric.
128 logLevel : `int, optional
129 Log level (an `logging` level constant, such as `logging.DEBUG`).
130 metadata : `collections.abc.MutableMapping`, optional
131 Metadata object to write entries to. Ignored if `None`.
132 logger : `logging.Logger`
133 Log object to write entries to. Ignored if `None`.
134 """
135 if obj is not None:
136 if metadata is None:
137 try:
138 metadata = obj.metadata
139 except AttributeError:
140 pass
141 if logger is None:
142 try:
143 logger = obj.log
144 except AttributeError:
145 pass
146 strList = []
147 for name, value in pairs:
148 if metadata is not None:
149 _add_to_metadata(metadata, name, value)
150 strList.append(f"{name}={value}")
151 if logger is not None:
152 # Want the file associated with this log message to be that
153 # of the caller.
154 stacklevel = _find_outside_stacklevel()
155 logging.getLogger("timer." + logger.name).log(logLevel, "; ".join(strList), stacklevel=stacklevel)
158def logInfo(obj: Any, prefix: str, logLevel: int = logging.DEBUG,
159 metadata: Optional[MutableMapping] = None, logger: Optional[logging.Logger] = None) -> None:
160 """Log timer information to ``obj.metadata`` and ``obj.log``.
162 Parameters
163 ----------
164 obj : `object`
165 An object with both or one these two attributes:
167 - ``metadata`` a `dict`-like container for storing metadata.
168 Can use the ``add(name, value)`` method if found, else will append
169 entries to a list.
170 - ``log`` an instance of `logging.Logger` or subclass.
172 If `None`, at least one of ``metadata`` or ``logger`` should be passed
173 or this function will do nothing.
174 prefix : `str`
175 Name prefix, the resulting entries are ``CpuTime``, etc.. For example
176 `timeMethod` uses ``prefix = Start`` when the method begins and
177 ``prefix = End`` when the method ends.
178 logLevel : optional
179 Log level (a `logging` level constant, such as `logging.DEBUG`).
180 metadata : `collections.abc.MutableMapping`, optional
181 Metadata object to write entries to, overriding ``obj.metadata``.
182 logger : `logging.Logger`
183 Log object to write entries to, overriding ``obj.log``.
185 Notes
186 -----
187 Logged items include:
189 - ``Utc``: UTC date in ISO format (only in metadata since log entries have
190 timestamps).
191 - ``CpuTime``: System + User CPU time (seconds). This should only be used
192 in differential measurements; the time reference point is undefined.
193 - ``MaxRss``: maximum resident set size.
195 All logged resource information is only for the current process; child
196 processes are excluded.
197 """
198 cpuTime = time.process_time()
199 res = resource.getrusage(resource.RUSAGE_SELF)
200 if metadata is None and obj is not None:
201 try:
202 metadata = obj.metadata
203 except AttributeError:
204 pass
205 if metadata is not None:
206 # Log messages already have timestamps.
207 utcStr = datetime.datetime.utcnow().isoformat()
208 _add_to_metadata(metadata, name=prefix + "Utc", value=utcStr)
209 logPairs(obj=obj,
210 pairs=[
211 (prefix + "CpuTime", cpuTime),
212 (prefix + "UserTime", res.ru_utime),
213 (prefix + "SystemTime", res.ru_stime),
214 (prefix + "MaxResidentSetSize", int(res.ru_maxrss)),
215 (prefix + "MinorPageFaults", int(res.ru_minflt)),
216 (prefix + "MajorPageFaults", int(res.ru_majflt)),
217 (prefix + "BlockInputs", int(res.ru_inblock)),
218 (prefix + "BlockOutputs", int(res.ru_oublock)),
219 (prefix + "VoluntaryContextSwitches", int(res.ru_nvcsw)),
220 (prefix + "InvoluntaryContextSwitches", int(res.ru_nivcsw)),
221 ],
222 logLevel=logLevel,
223 metadata=metadata,
224 logger=logger)
227def timeMethod(_func: Optional[Any] = None, *, metadata: Optional[MutableMapping] = None,
228 logger: Optional[logging.Logger] = None,
229 logLevel: int = logging.DEBUG) -> Callable:
230 """Decorator to measure duration of a method.
232 Parameters
233 ----------
234 func
235 The method to wrap.
236 metadata : `collections.abc.MutableMapping`, optional
237 Metadata to use as override if the instance object attached
238 to this timer does not support a ``metadata`` property.
239 logger : `logging.Logger`, optional
240 Logger to use when the class containing the decorated method does not
241 have a ``log`` property.
242 logLevel : `int`, optional
243 Log level to use when logging messages. Default is `~logging.DEBUG`.
245 Notes
246 -----
247 Writes various measures of time and possibly memory usage to the
248 metadata; all items are prefixed with the function name.
250 .. warning::
252 This decorator only works with instance methods of any class
253 with these attributes:
255 - ``metadata``: an instance of `collections.abc.Mapping`. The ``add``
256 method will be used if available, else entries will be added to a
257 list.
258 - ``log``: an instance of `logging.Logger` or subclass.
260 Examples
261 --------
262 To use:
264 .. code-block:: python
266 import lsst.utils as utils
267 import lsst.pipe.base as pipeBase
268 class FooTask(pipeBase.Task):
269 pass
271 @utils.timeMethod
272 def run(self, ...): # or any other instance method you want to time
273 pass
274 """
276 def decorator_timer(func: Callable) -> Callable:
277 @functools.wraps(func)
278 def wrapper(self: Any, *args: Any, **keyArgs: Any) -> Any:
279 logInfo(obj=self, prefix=func.__name__ + "Start", metadata=metadata, logger=logger,
280 logLevel=logLevel)
281 try:
282 res = func(self, *args, **keyArgs)
283 finally:
284 logInfo(obj=self, prefix=func.__name__ + "End", metadata=metadata, logger=logger,
285 logLevel=logLevel)
286 return res
287 return wrapper
289 if _func is None:
290 return decorator_timer
291 else:
292 return decorator_timer(_func)
295@contextmanager
296def time_this(log: Optional[LsstLoggers] = None, msg: Optional[str] = None,
297 level: int = logging.DEBUG, prefix: Optional[str] = "timer",
298 args: Iterable[Any] = ()) -> Iterator[None]:
299 """Time the enclosed block and issue a log message.
301 Parameters
302 ----------
303 log : `logging.Logger`, optional
304 Logger to use to report the timer message. The root logger will
305 be used if none is given.
306 msg : `str`, optional
307 Context to include in log message.
308 level : `int`, optional
309 Python logging level to use to issue the log message. If the
310 code block raises an exception the log message will automatically
311 switch to level ERROR.
312 prefix : `str`, optional
313 Prefix to use to prepend to the supplied logger to
314 create a new logger to use instead. No prefix is used if the value
315 is set to `None`. Defaults to "timer".
316 args : iterable of any
317 Additional parameters passed to the log command that should be
318 written to ``msg``.
319 """
320 if log is None:
321 log = logging.getLogger()
322 if prefix:
323 log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix
324 log = logging.getLogger(log_name)
326 success = False
327 start = time.time()
328 try:
329 yield
330 success = True
331 finally:
332 end = time.time()
334 # The message is pre-inserted to allow the logger to expand
335 # the additional args provided. Make that easier by converting
336 # the None message to empty string.
337 if msg is None:
338 msg = ""
340 if not success:
341 # Something went wrong so change the log level to indicate
342 # this.
343 level = logging.ERROR
345 # Specify stacklevel to ensure the message is reported from the
346 # caller (1 is this file, 2 is contextlib, 3 is user)
347 log.log(level, msg + "%sTook %.4f seconds", *args,
348 ": " if msg else "", end - start, stacklevel=3)