Coverage for python/lsst/utils/timer.py: 19%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11#
13"""Utilities for measuring execution time.
14"""
16from __future__ import annotations
18__all__ = ["logInfo", "timeMethod", "time_this"]
20import functools
21import logging
22import resource
23import time
24import datetime
25import inspect
26from contextlib import contextmanager
28from typing import (
29 Any,
30 Callable,
31 Collection,
32 Iterable,
33 Iterator,
34 MutableMapping,
35 Optional,
36 Tuple,
37 TYPE_CHECKING,
38)
40if TYPE_CHECKING: 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true
41 from .logging import LsstLoggers
44def _add_to_metadata(metadata: MutableMapping, name: str, value: Any) -> None:
45 """Add a value to dict-like object, creating list as needed.
47 The list grows as more values are added for that key.
49 Parameters
50 ----------
51 metadata : `dict`-like, optional
52 `dict`-like object that can store keys. Uses `add()` method if
53 one is available, else creates list and appends value if needed.
54 name : `str`
55 The key to use in the metadata dictionary.
56 value : Any
57 Value to store in the list.
58 """
59 try:
60 try:
61 # PropertySet should always prefer LongLong for integers
62 metadata.addLongLong(name, value) # type: ignore
63 except (TypeError, AttributeError):
64 metadata.add(name, value) # type: ignore
65 except AttributeError:
66 pass
67 else:
68 return
70 # Fallback code where `add` is not implemented.
71 if name not in metadata:
72 metadata[name] = []
73 metadata[name].append(value)
76def _find_outside_stacklevel() -> int:
77 """Find the stack level corresponding to caller code outside of this
78 module.
80 This can be passed directly to `logging.Logger.log()` to ensure
81 that log messages are issued as if they are coming from caller code.
83 Returns
84 -------
85 stacklevel : `int`
86 The stack level to use to refer to a caller outside of this module.
87 A ``stacklevel`` of ``1`` corresponds to the caller of this internal
88 function and that is the default expected by `logging.Logger.log()`.
90 Notes
91 -----
92 Intended to be called from the function that is going to issue a log
93 message. The result should be passed into `~logging.Logger.log` via the
94 keyword parameter ``stacklevel``.
95 """
96 stacklevel = 1 # the default for `Logger.log`
97 for i, s in enumerate(inspect.stack()):
98 module = inspect.getmodule(s.frame)
99 if module is None:
100 # Stack frames sometimes hang around so explicilty delete.
101 del s
102 continue
103 if not module.__name__.startswith("lsst.utils"):
104 # 0 will be this function.
105 # 1 will be the caller which will be the default for `Logger.log`
106 # and so does not need adjustment.
107 stacklevel = i
108 break
109 # Stack frames sometimes hang around so explicilty delete.
110 del s
112 return stacklevel
115def logPairs(obj: Any, pairs: Collection[Tuple[str, Any]], logLevel: int = logging.DEBUG,
116 metadata: Optional[MutableMapping] = None,
117 logger: Optional[logging.Logger] = None, stacklevel: Optional[int] = None) -> None:
118 """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``
120 Parameters
121 ----------
122 obj : `object`
123 An object with one or both of these two attributes:
125 - ``metadata`` a `dict`-like container for storing metadata.
126 Can use the ``add(name, value)`` method if found, else will append
127 entries to a list.
128 - ``log`` an instance of `logging.Logger` or subclass.
130 If `None`, at least one of ``metadata`` or ``logger`` should be passed
131 or this function will do nothing.
132 pairs : sequence
133 A sequence of ``(name, value)`` pairs, with value typically numeric.
134 logLevel : `int, optional
135 Log level (an `logging` level constant, such as `logging.DEBUG`).
136 metadata : `collections.abc.MutableMapping`, optional
137 Metadata object to write entries to. Ignored if `None`.
138 logger : `logging.Logger`
139 Log object to write entries to. Ignored if `None`.
140 stacklevel : `int`, optional
141 The stack level to pass to the logger to adjust which stack frame
142 is used to report the file information. If `None` the stack level
143 is computed such that it is reported as the first package outside
144 of the utils package. If a value is given here it is adjusted by
145 1 to account for this caller.
146 """
147 if obj is not None:
148 if metadata is None:
149 try:
150 metadata = obj.metadata
151 except AttributeError:
152 pass
153 if logger is None:
154 try:
155 logger = obj.log
156 except AttributeError:
157 pass
158 strList = []
159 for name, value in pairs:
160 if metadata is not None:
161 _add_to_metadata(metadata, name, value)
162 strList.append(f"{name}={value}")
163 if logger is not None:
164 # Want the file associated with this log message to be that
165 # of the caller. This is expensive so only do it if we know the
166 # message will be issued.
167 timer_logger = logging.getLogger("timer." + logger.name)
168 if timer_logger.isEnabledFor(logLevel):
169 if stacklevel is None:
170 stacklevel = _find_outside_stacklevel()
171 else:
172 # Account for the caller stack.
173 stacklevel += 1
174 timer_logger.log(logLevel, "; ".join(strList), stacklevel=stacklevel)
177def logInfo(obj: Any, prefix: str, logLevel: int = logging.DEBUG,
178 metadata: Optional[MutableMapping] = None, logger: Optional[logging.Logger] = None,
179 stacklevel: Optional[int] = None) -> None:
180 """Log timer information to ``obj.metadata`` and ``obj.log``.
182 Parameters
183 ----------
184 obj : `object`
185 An object with both or one these two attributes:
187 - ``metadata`` a `dict`-like container for storing metadata.
188 Can use the ``add(name, value)`` method if found, else will append
189 entries to a list.
190 - ``log`` an instance of `logging.Logger` or subclass.
192 If `None`, at least one of ``metadata`` or ``logger`` should be passed
193 or this function will do nothing.
194 prefix : `str`
195 Name prefix, the resulting entries are ``CpuTime``, etc.. For example
196 `timeMethod` uses ``prefix = Start`` when the method begins and
197 ``prefix = End`` when the method ends.
198 logLevel : optional
199 Log level (a `logging` level constant, such as `logging.DEBUG`).
200 metadata : `collections.abc.MutableMapping`, optional
201 Metadata object to write entries to, overriding ``obj.metadata``.
202 logger : `logging.Logger`
203 Log object to write entries to, overriding ``obj.log``.
204 stacklevel : `int`, optional
205 The stack level to pass to the logger to adjust which stack frame
206 is used to report the file information. If `None` the stack level
207 is computed such that it is reported as the first package outside
208 of the utils package. If a value is given here it is adjusted by
209 1 to account for this caller.
211 Notes
212 -----
213 Logged items include:
215 - ``Utc``: UTC date in ISO format (only in metadata since log entries have
216 timestamps).
217 - ``CpuTime``: System + User CPU time (seconds). This should only be used
218 in differential measurements; the time reference point is undefined.
219 - ``MaxRss``: maximum resident set size.
221 All logged resource information is only for the current process; child
222 processes are excluded.
223 """
224 cpuTime = time.process_time()
225 res = resource.getrusage(resource.RUSAGE_SELF)
226 if metadata is None and obj is not None:
227 try:
228 metadata = obj.metadata
229 except AttributeError:
230 pass
231 if metadata is not None:
232 # Log messages already have timestamps.
233 utcStr = datetime.datetime.utcnow().isoformat()
234 _add_to_metadata(metadata, name=prefix + "Utc", value=utcStr)
235 if stacklevel is not None:
236 # Account for the caller of this routine not knowing that we
237 # are going one down in the stack.
238 stacklevel += 1
239 logPairs(obj=obj,
240 pairs=[
241 (prefix + "CpuTime", cpuTime),
242 (prefix + "UserTime", res.ru_utime),
243 (prefix + "SystemTime", res.ru_stime),
244 (prefix + "MaxResidentSetSize", int(res.ru_maxrss)),
245 (prefix + "MinorPageFaults", int(res.ru_minflt)),
246 (prefix + "MajorPageFaults", int(res.ru_majflt)),
247 (prefix + "BlockInputs", int(res.ru_inblock)),
248 (prefix + "BlockOutputs", int(res.ru_oublock)),
249 (prefix + "VoluntaryContextSwitches", int(res.ru_nvcsw)),
250 (prefix + "InvoluntaryContextSwitches", int(res.ru_nivcsw)),
251 ],
252 logLevel=logLevel,
253 metadata=metadata,
254 logger=logger,
255 stacklevel=stacklevel)
258def timeMethod(_func: Optional[Any] = None, *, metadata: Optional[MutableMapping] = None,
259 logger: Optional[logging.Logger] = None,
260 logLevel: int = logging.DEBUG) -> Callable:
261 """Decorator to measure duration of a method.
263 Parameters
264 ----------
265 func
266 The method to wrap.
267 metadata : `collections.abc.MutableMapping`, optional
268 Metadata to use as override if the instance object attached
269 to this timer does not support a ``metadata`` property.
270 logger : `logging.Logger`, optional
271 Logger to use when the class containing the decorated method does not
272 have a ``log`` property.
273 logLevel : `int`, optional
274 Log level to use when logging messages. Default is `~logging.DEBUG`.
276 Notes
277 -----
278 Writes various measures of time and possibly memory usage to the
279 metadata; all items are prefixed with the function name.
281 .. warning::
283 This decorator only works with instance methods of any class
284 with these attributes:
286 - ``metadata``: an instance of `collections.abc.Mapping`. The ``add``
287 method will be used if available, else entries will be added to a
288 list.
289 - ``log``: an instance of `logging.Logger` or subclass.
291 Examples
292 --------
293 To use:
295 .. code-block:: python
297 import lsst.utils as utils
298 import lsst.pipe.base as pipeBase
299 class FooTask(pipeBase.Task):
300 pass
302 @utils.timeMethod
303 def run(self, ...): # or any other instance method you want to time
304 pass
305 """
307 def decorator_timer(func: Callable) -> Callable:
308 @functools.wraps(func)
309 def timeMethod_wrapper(self: Any, *args: Any, **keyArgs: Any) -> Any:
310 # Adjust the stacklevel to account for the wrappers.
311 # stacklevel 1 would make the log message come from this function
312 # but we want it to come from the file that defined the method
313 # so need to increment by 1 to get to the caller.
314 stacklevel = 2
315 logInfo(obj=self, prefix=func.__name__ + "Start", metadata=metadata, logger=logger,
316 logLevel=logLevel, stacklevel=stacklevel)
317 try:
318 res = func(self, *args, **keyArgs)
319 finally:
320 logInfo(obj=self, prefix=func.__name__ + "End", metadata=metadata, logger=logger,
321 logLevel=logLevel, stacklevel=stacklevel)
322 return res
323 return timeMethod_wrapper
325 if _func is None:
326 return decorator_timer
327 else:
328 return decorator_timer(_func)
331@contextmanager
332def time_this(log: Optional[LsstLoggers] = None, msg: Optional[str] = None,
333 level: int = logging.DEBUG, prefix: Optional[str] = "timer",
334 args: Iterable[Any] = ()) -> Iterator[None]:
335 """Time the enclosed block and issue a log message.
337 Parameters
338 ----------
339 log : `logging.Logger`, optional
340 Logger to use to report the timer message. The root logger will
341 be used if none is given.
342 msg : `str`, optional
343 Context to include in log message.
344 level : `int`, optional
345 Python logging level to use to issue the log message. If the
346 code block raises an exception the log message will automatically
347 switch to level ERROR.
348 prefix : `str`, optional
349 Prefix to use to prepend to the supplied logger to
350 create a new logger to use instead. No prefix is used if the value
351 is set to `None`. Defaults to "timer".
352 args : iterable of any
353 Additional parameters passed to the log command that should be
354 written to ``msg``.
355 """
356 if log is None:
357 log = logging.getLogger()
358 if prefix:
359 log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix
360 log = logging.getLogger(log_name)
362 success = False
363 start = time.time()
364 try:
365 yield
366 success = True
367 finally:
368 end = time.time()
370 # The message is pre-inserted to allow the logger to expand
371 # the additional args provided. Make that easier by converting
372 # the None message to empty string.
373 if msg is None:
374 msg = ""
376 if not success:
377 # Something went wrong so change the log level to indicate
378 # this.
379 level = logging.ERROR
381 # Specify stacklevel to ensure the message is reported from the
382 # caller (1 is this file, 2 is contextlib, 3 is user)
383 log.log(level, msg + "%sTook %.4f seconds", *args,
384 ": " if msg else "", end - start, stacklevel=3)