Coverage for python/lsst/utils/logging.py: 49%
112 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-01 15:14 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-01 15:14 -0700
1# This file is part of utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = (
15 "TRACE",
16 "VERBOSE",
17 "getLogger",
18 "getTraceLogger",
19 "LsstLogAdapter",
20 "LsstLoggers",
21 "PeriodicLogger",
22 "trace_set_at",
23)
25import logging
26import sys
27import time
28from collections.abc import Generator
29from contextlib import contextmanager
30from logging import LoggerAdapter
31from typing import Any, TypeAlias, Union
33try:
34 import lsst.log.utils as logUtils
35except ImportError:
36 logUtils = None
39# log level for trace (verbose debug).
40TRACE = 5
41logging.addLevelName(TRACE, "TRACE")
43# Verbose logging is midway between INFO and DEBUG.
44VERBOSE = (logging.INFO + logging.DEBUG) // 2
45logging.addLevelName(VERBOSE, "VERBOSE")
48def _calculate_base_stacklevel(default: int, offset: int) -> int:
49 """Calculate the default logging stacklevel to use.
51 Parameters
52 ----------
53 default : `int`
54 The stacklevel to use in Python 3.11 and newer where the only
55 thing to take into account is the number of levels above the core
56 Python logging infrastructure.
57 offset : `int`
58 The offset to apply for older Python implementations that need to
59 take into account internal call stacks.
61 Returns
62 -------
63 stacklevel : `int`
64 The stack level to pass to internal logging APIs that should result
65 in the log messages being reported in caller lines.
67 Notes
68 -----
69 In Python 3.11 the logging infrastructure was fixed such that we no
70 longer need to understand that a LoggerAdapter log messages need to
71 have a stack level one higher than a Logger would need. ``stacklevel=1``
72 now always means "log from the caller's line" without the caller having
73 to understand internal implementation details.
74 """
75 stacklevel = default
76 if sys.version_info < (3, 11, 0): 76 ↛ 77line 76 didn't jump to line 77, because the condition on line 76 was never true
77 stacklevel += offset
78 return stacklevel
81def trace_set_at(name: str, number: int) -> None:
82 """Adjust logging level to display messages with the trace number being
83 less than or equal to the provided value.
85 Parameters
86 ----------
87 name : `str`
88 Name of the logger.
89 number : `int`
90 The trace number threshold for display.
92 Examples
93 --------
94 .. code-block:: python
96 lsst.utils.logging.trace_set_at("lsst.afw", 3)
98 This will set loggers ``TRACE0.lsst.afw`` to ``TRACE3.lsst.afw`` to
99 ``DEBUG`` and ``TRACE4.lsst.afw`` and ``TRACE5.lsst.afw`` to ``INFO``.
101 Notes
102 -----
103 Loggers ``TRACE0.`` to ``TRACE5.`` are set. All loggers above
104 the specified threshold are set to ``INFO`` and those below the threshold
105 are set to ``DEBUG``. The expectation is that ``TRACE`` loggers only
106 issue ``DEBUG`` log messages.
108 If ``lsst.log`` is installed, this function will also call
109 `lsst.log.utils.traceSetAt` to ensure that non-Python loggers are
110 also configured correctly.
111 """
112 for i in range(6):
113 level = logging.INFO if i > number else logging.DEBUG
114 getTraceLogger(name, i).setLevel(level)
116 # if lsst log is available also set the trace loggers there.
117 if logUtils is not None:
118 logUtils.traceSetAt(name, number)
121class _F:
122 """Format, supporting `str.format()` syntax.
124 Notes
125 -----
126 This follows the recommendation from
127 https://docs.python.org/3/howto/logging-cookbook.html#using-custom-message-objects
128 """
130 def __init__(self, fmt: str, /, *args: Any, **kwargs: Any):
131 self.fmt = fmt
132 self.args = args
133 self.kwargs = kwargs
135 def __str__(self) -> str:
136 return self.fmt.format(*self.args, **self.kwargs)
139class LsstLogAdapter(LoggerAdapter):
140 """A special logging adapter to provide log features for LSST code.
142 Expected to be instantiated initially by a call to `getLogger()`.
144 This class provides enhancements over `logging.Logger` that include:
146 * Methods for issuing trace and verbose level log messages.
147 * Provision of a context manager to temporarily change the log level.
148 * Attachment of logging level constants to the class to make it easier
149 for a Task writer to access a specific log level without having to
150 know the underlying logger class.
151 """
153 # Store logging constants in the class for convenience. This is not
154 # something supported by Python loggers but can simplify some
155 # logic if the logger is available.
156 CRITICAL = logging.CRITICAL
157 ERROR = logging.ERROR
158 DEBUG = logging.DEBUG
159 INFO = logging.INFO
160 WARNING = logging.WARNING
162 # Python supports these but prefers they are not used.
163 FATAL = logging.FATAL
164 WARN = logging.WARN
166 # These are specific to Tasks
167 TRACE = TRACE
168 VERBOSE = VERBOSE
170 # The stack level to use when issuing log messages. For Python 3.11
171 # this is generally 2 (this method and the internal infrastructure).
172 # For older python we need one higher because of the extra indirection
173 # via LoggingAdapter internals.
174 _stacklevel = _calculate_base_stacklevel(2, 1)
176 @contextmanager
177 def temporary_log_level(self, level: int | str) -> Generator:
178 """Temporarily set the level of this logger.
180 Parameters
181 ----------
182 level : `int`
183 The new temporary log level.
184 """
185 old = self.level
186 self.setLevel(level)
187 try:
188 yield
189 finally:
190 self.setLevel(old)
192 @property
193 def level(self) -> int:
194 """Return current level of this logger (``int``)."""
195 return self.logger.level
197 def getChild(self, name: str) -> LsstLogAdapter:
198 """Get the named child logger.
200 Parameters
201 ----------
202 name : `str`
203 Name of the child relative to this logger.
205 Returns
206 -------
207 child : `LsstLogAdapter`
208 The child logger.
209 """
210 return getLogger(name=name, logger=self.logger)
212 def _process_stacklevel(self, kwargs: dict[str, Any], offset: int = 0) -> int:
213 # Return default stacklevel, taking into account kwargs[stacklevel].
214 stacklevel = self._stacklevel
215 if "stacklevel" in kwargs:
216 # External user expects stacklevel=1 to mean "report from their
217 # line" but the code here is already trying to achieve that by
218 # default. Therefore if an external stacklevel is specified we
219 # adjust their stacklevel request by 1.
220 stacklevel = stacklevel + kwargs.pop("stacklevel") - 1
222 # The offset can be used to indicate that we have to take into account
223 # additional internal layers before calling python logging.
224 return _calculate_base_stacklevel(stacklevel, offset)
226 def fatal(self, msg: str, *args: Any, **kwargs: Any) -> None:
227 # Python does not provide this method in LoggerAdapter but does
228 # not formally deprecate it in favor of critical() either.
229 # Provide it without deprecation message for consistency with Python.
230 # Have to adjust stacklevel on Python 3.10 and older to account
231 # for call through self.critical.
232 stacklevel = self._process_stacklevel(kwargs, offset=1)
233 self.critical(msg, *args, **kwargs, stacklevel=stacklevel)
235 def verbose(self, fmt: str, *args: Any, **kwargs: Any) -> None:
236 """Issue a VERBOSE level log message.
238 Arguments are as for `logging.info`.
239 ``VERBOSE`` is between ``DEBUG`` and ``INFO``.
241 Parameters
242 ----------
243 fmt : `str`
244 Log message.
245 *args : `~typing.Any`
246 Parameters references by log message.
247 **kwargs : `~typing.Any`
248 Parameters forwarded to `log`.
249 """
250 # There is no other way to achieve this other than a special logger
251 # method.
252 # Stacklevel is passed in so that the correct line is reported
253 stacklevel = self._process_stacklevel(kwargs)
254 self.log(VERBOSE, fmt, *args, **kwargs, stacklevel=stacklevel)
256 def trace(self, fmt: str, *args: Any, **kwargs: Any) -> None:
257 """Issue a TRACE level log message.
259 Arguments are as for `logging.info`.
260 ``TRACE`` is lower than ``DEBUG``.
262 Parameters
263 ----------
264 fmt : `str`
265 Log message.
266 *args : `~typing.Any`
267 Parameters references by log message.
268 **kwargs : `~typing.Any`
269 Parameters forwarded to `log`.
270 """
271 # There is no other way to achieve this other than a special logger
272 # method.
273 stacklevel = self._process_stacklevel(kwargs)
274 self.log(TRACE, fmt, *args, **kwargs, stacklevel=stacklevel)
276 def setLevel(self, level: int | str) -> None:
277 """Set the level for the logger, trapping lsst.log values.
279 Parameters
280 ----------
281 level : `int`
282 The level to use. If the level looks too big to be a Python
283 logging level it is assumed to be a lsst.log level.
284 """
285 if isinstance(level, int) and level > logging.CRITICAL:
286 self.logger.warning(
287 "Attempting to set level to %d -- looks like an lsst.log level so scaling it accordingly.",
288 level,
289 )
290 level //= 1000
292 self.logger.setLevel(level)
294 @property
295 def handlers(self) -> list[logging.Handler]:
296 """Log handlers associated with this logger."""
297 return self.logger.handlers
299 def addHandler(self, handler: logging.Handler) -> None:
300 """Add a handler to this logger.
302 Parameters
303 ----------
304 handler : `logging.Handler`
305 Handler to add. The handler is forwarded to the underlying logger.
306 """
307 self.logger.addHandler(handler)
309 def removeHandler(self, handler: logging.Handler) -> None:
310 """Remove the given handler from the underlying logger.
312 Parameters
313 ----------
314 handler : `logging.Handler`
315 Handler to remove.
316 """
317 self.logger.removeHandler(handler)
320def getLogger(name: str | None = None, logger: logging.Logger | None = None) -> LsstLogAdapter:
321 """Get a logger compatible with LSST usage.
323 Parameters
324 ----------
325 name : `str`, optional
326 Name of the logger. Root logger if `None`.
327 logger : `logging.Logger` or `LsstLogAdapter`
328 If given the logger is converted to the relevant logger class.
329 If ``name`` is given the logger is assumed to be a child of the
330 supplied logger.
332 Returns
333 -------
334 logger : `LsstLogAdapter`
335 The relevant logger.
337 Notes
338 -----
339 A `logging.LoggerAdapter` is used since it is easier to provide a more
340 uniform interface than when using `logging.setLoggerClass`. An adapter
341 can be wrapped around the root logger and the `~logging.setLoggerClass`
342 will return the logger first given that name even if the name was
343 used before the `~lsst.pipe.base.Task` was created.
344 """
345 if not logger:
346 logger = logging.getLogger(name)
347 elif name:
348 logger = logger.getChild(name)
349 return LsstLogAdapter(logger, {})
352LsstLoggers: TypeAlias = Union[logging.Logger, LsstLogAdapter]
355def getTraceLogger(logger: str | LsstLoggers, trace_level: int) -> LsstLogAdapter:
356 """Get a logger with the appropriate TRACE name.
358 Parameters
359 ----------
360 logger : `logging.Logger` or `LsstLogAdapter` or `lsst.log.Log` or `str`
361 A logger to be used to derive the new trace logger. Can be a logger
362 object that supports the ``name`` property or a string.
363 trace_level : `int`
364 The trace level to use for the logger.
366 Returns
367 -------
368 trace_logger : `LsstLogAdapter`
369 A new trace logger. The name will be derived by pre-pending ``TRACEn.``
370 to the name of the supplied logger. If the root logger is given
371 the returned logger will be named ``TRACEn``.
372 """
373 name = getattr(logger, "name", str(logger))
374 trace_name = f"TRACE{trace_level}.{name}" if name else f"TRACE{trace_level}"
375 return getLogger(trace_name)
378class PeriodicLogger:
379 """Issue log messages if a time threshold has elapsed.
381 This class can be used in long-running sections of code where it would
382 be useful to issue a log message periodically to show that the
383 algorithm is progressing.
385 Parameters
386 ----------
387 logger : `logging.Logger` or `LsstLogAdapter`
388 Logger to use when issuing a message.
389 interval : `float`
390 The minimum interval between log messages. If `None` the class
391 default will be used.
392 level : `int`, optional
393 Log level to use when issuing messages.
394 """
396 LOGGING_INTERVAL = 600.0
397 """Default interval between log messages."""
399 def __init__(self, logger: LsstLoggers, interval: float | None = None, level: int = VERBOSE):
400 self.logger = logger
401 self.interval = interval if interval is not None else self.LOGGING_INTERVAL
402 self.level = level
403 self.next_log_time = time.time() + self.interval
404 self.num_issued = 0
406 # The stacklevel we need to issue logs is determined by the type
407 # of logger we have been given. A LoggerAdapter has an extra
408 # level of indirection. In Python 3.11 the logging infrastructure
409 # takes care to check for internal logging stack frames so there
410 # is no need for a difference.
411 self._stacklevel = _calculate_base_stacklevel(2, 1 if isinstance(self.logger, LoggerAdapter) else 0)
413 def log(self, msg: str, *args: Any) -> bool:
414 """Issue a log message if the interval has elapsed.
416 Parameters
417 ----------
418 msg : `str`
419 Message to issue if the time has been exceeded.
420 *args : Any
421 Parameters to be passed to the log system.
423 Returns
424 -------
425 issued : `bool`
426 Returns `True` if a log message was sent to the logging system.
427 Returns `False` if the interval has not yet elapsed. Returning
428 `True` does not indicate whether the log message was in fact
429 issued by the logging system.
430 """
431 if (current_time := time.time()) > self.next_log_time:
432 self.logger.log(self.level, msg, *args, stacklevel=self._stacklevel)
433 self.next_log_time = current_time + self.interval
434 self.num_issued += 1
435 return True
436 return False