Coverage for python/lsst/daf/butler/cli/cliLog.py: 30%
159 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "PrecisionLogFormatter",
31 "CliLog",
32)
34import datetime
35import logging
36import os
37from typing import Any
39try:
40 import lsst.log as lsstLog
41except ModuleNotFoundError:
42 lsstLog = None
44from lsst.utils.logging import TRACE, VERBOSE
46from ..core.logging import ButlerMDC, JsonLogFormatter
49class PrecisionLogFormatter(logging.Formatter):
50 """A log formatter that issues accurate timezone-aware timestamps."""
52 converter = datetime.datetime.fromtimestamp # type: ignore
54 use_local = True
55 """Control whether local time is displayed instead of UTC."""
57 def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str:
58 """Format the time as an aware datetime."""
59 ct: datetime.datetime = self.converter(record.created, tz=datetime.timezone.utc) # type: ignore
60 if self.use_local:
61 ct = ct.astimezone()
62 if datefmt:
63 s = ct.strftime(datefmt)
64 else:
65 s = ct.isoformat(sep="T", timespec="milliseconds")
66 return s
69class CliLog:
70 """Interface for managing python logging and ``lsst.log``.
72 This class defines log format strings for the log output and timestamp
73 formats. It also configures ``lsst.log`` to forward all messages to
74 Python `logging`.
76 This class can perform log uninitialization, which allows command line
77 interface code that initializes logging to run unit tests that execute in
78 batches, without affecting other unit tests. See ``resetLog``.
79 """
81 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
83 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
84 """The log format used when the lsst.log package is not importable and the
85 log is initialized with longlog=True."""
87 pylog_normalFmt = "{name} {levelname}: {message}"
88 """The log format used when the lsst.log package is not importable and the
89 log is initialized with longlog=False."""
91 configState: list[tuple[Any, ...]] = []
92 """Configuration state. Contains tuples where first item in a tuple is
93 a method and remaining items are arguments for the method.
94 """
96 _initialized = False
97 _componentSettings: list[ComponentSettings] = []
99 _fileHandlers: list[logging.FileHandler] = []
100 """Any FileHandler classes attached to the root logger by this class
101 that need to be closed on reset."""
103 @staticmethod
104 def root_loggers() -> set[str]:
105 """Return the default root logger.
107 Returns
108 -------
109 log_name : `set` of `str`
110 The name(s) of the root logger(s) to use when the log level is
111 being set without a log name being specified.
113 Notes
114 -----
115 The default is ``lsst`` (which controls the butler infrastructure)
116 but additional loggers can be specified by setting the environment
117 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain
118 multiple default loggers separated by a ``:``.
119 """
120 log_names = {"lsst"}
121 envvar = "DAF_BUTLER_ROOT_LOGGER"
122 if envvar in os.environ: 122 ↛ 123line 122 didn't jump to line 123, because the condition on line 122 was never true
123 log_names |= set(os.environ[envvar].split(":"))
124 return log_names
126 @classmethod
127 def initLog(
128 cls,
129 longlog: bool,
130 log_tty: bool = True,
131 log_file: tuple[str, ...] = (),
132 log_label: dict[str, str] | None = None,
133 ) -> None:
134 """Initialize logging. This should only be called once per program
135 execution. After the first call this will log a warning and return.
137 If lsst.log is importable, will add its log handler to the python
138 root logger's handlers.
140 Parameters
141 ----------
142 longlog : `bool`
143 If True, make log messages appear in long format, by default False.
144 log_tty : `bool`
145 Control whether a default stream handler is enabled that logs
146 to the terminal.
147 log_file : `tuple` of `str`
148 Path to files to write log records. If path ends in ``.json`` the
149 records will be written in JSON format. Else they will be written
150 in text format. If empty no log file will be created. Records
151 will be appended to this file if it exists.
152 log_label : `dict` of `str`
153 Keys and values to be stored in logging MDC for all JSON log
154 records. Keys will be upper-cased.
155 """
156 if cls._initialized:
157 # Unit tests that execute more than one command do end up
158 # calling this function multiple times in one program execution,
159 # so do log a debug but don't log an error or fail, just make the
160 # re-initialization a no-op.
161 log = logging.getLogger(__name__)
162 log.debug("Log is already initialized, returning without re-initializing.")
163 return
164 cls._initialized = True
165 cls._recordComponentSetting(None)
167 if lsstLog is not None:
168 # Ensure that log messages are forwarded back to python.
169 # Disable use of lsst.log MDC -- we expect butler uses to
170 # use ButlerMDC.
171 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
173 # Forward python lsst.log messages directly to python logging.
174 # This can bypass the C++ layer entirely but requires that
175 # MDC is set via ButlerMDC, rather than in lsst.log.
176 lsstLog.usePythonLogging()
178 formatter: logging.Formatter
179 if not log_tty:
180 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
181 elif longlog:
182 # Want to create our own Formatter so that we can get high
183 # precision timestamps. This requires we attach our own
184 # default stream handler.
185 defaultHandler = logging.StreamHandler()
186 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
187 defaultHandler.setFormatter(formatter)
189 logging.basicConfig(
190 level=logging.WARNING,
191 force=True,
192 handlers=[defaultHandler],
193 )
195 else:
196 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{")
198 # Initialize the root logger. Calling this ensures that both
199 # python loggers and lsst loggers are consistent in their default
200 # logging level.
201 cls._setLogLevel(".", "WARNING")
203 # Initialize default root logger level.
204 cls._setLogLevel(None, "INFO")
206 # also capture warnings and send them to logging
207 logging.captureWarnings(True)
209 # Create a record factory that ensures that an MDC is attached
210 # to the records. By default this is only used for long-log
211 # but always enable it for when someone adds a new handler
212 # that needs it.
213 ButlerMDC.add_mdc_log_record_factory()
215 # Set up the file logger
216 for file in log_file:
217 handler = logging.FileHandler(file)
218 if file.endswith(".json"):
219 formatter = JsonLogFormatter()
220 else:
221 if longlog:
222 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
223 else:
224 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
225 handler.setFormatter(formatter)
226 logging.getLogger().addHandler(handler)
227 cls._fileHandlers.append(handler)
229 # Add any requested MDC records.
230 if log_label:
231 for key, value in log_label.items():
232 ButlerMDC.MDC(key.upper(), value)
234 # remember this call
235 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
237 @classmethod
238 def resetLog(cls) -> None:
239 """Uninitialize the butler CLI Log handler and reset component log
240 levels.
242 If the lsst.log handler was added to the python root logger's handlers
243 in `initLog`, it will be removed here.
245 For each logger level that was set by this class, sets that logger's
246 level to the value it was before this class set it. For lsst.log, if a
247 component level was uninitialized, it will be set to
248 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
249 component back to an uninitialized state.
250 """
251 if lsstLog:
252 lsstLog.doNotUsePythonLogging()
253 for componentSetting in reversed(cls._componentSettings):
254 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
255 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
256 logger = logging.getLogger(componentSetting.component)
257 logger.setLevel(componentSetting.pythonLogLevel)
258 cls._setLogLevel(None, "INFO")
260 ButlerMDC.restore_log_record_factory()
262 # Remove the FileHandler we may have attached.
263 root = logging.getLogger()
264 for handler in cls._fileHandlers:
265 handler.close()
266 root.removeHandler(handler)
268 cls._fileHandlers.clear()
269 cls._initialized = False
270 cls.configState = []
272 @classmethod
273 def setLogLevels(cls, logLevels: list[tuple[str | None, str]] | dict[str, str]) -> None:
274 """Set log level for one or more components or the root logger.
276 Parameters
277 ----------
278 logLevels : `list` of `tuple`
279 per-component logging levels, each item in the list is a tuple
280 (component, level), `component` is a logger name or an empty string
281 or `None` for default root logger, `level` is a logging level name,
282 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
284 Notes
285 -----
286 The special name ``.`` can be used to set the Python root
287 logger.
288 """
289 if isinstance(logLevels, dict):
290 logLevels = list(logLevels.items())
292 # configure individual loggers
293 for component, level in logLevels:
294 cls._setLogLevel(component, level)
295 # remember this call
296 cls.configState.append((cls._setLogLevel, component, level))
298 @classmethod
299 def _setLogLevel(cls, component: str | None, level: str) -> None:
300 """Set the log level for the given component. Record the current log
301 level of the component so that it can be restored when resetting this
302 log.
304 Parameters
305 ----------
306 component : `str` or None
307 The name of the log component or None for the default logger.
308 The root logger can be specified either by an empty string or
309 with the special name ``.``.
310 level : `str`
311 A valid python logging level.
312 """
313 components: set[str | None]
314 if component is None:
315 components = set(cls.root_loggers())
316 elif not component or component == ".":
317 components = {None}
318 else:
319 components = {component}
320 for component in components:
321 cls._recordComponentSetting(component)
322 if lsstLog is not None:
323 lsstLogger = lsstLog.Log.getLogger(component or "")
324 lsstLogger.setLevel(cls._getLsstLogLevel(level))
325 pylevel = cls._getPyLogLevel(level)
326 if pylevel is not None:
327 logging.getLogger(component or None).setLevel(pylevel)
329 @staticmethod
330 def _getPyLogLevel(level: str) -> int | None:
331 """Get the numeric value for the given log level name.
333 Parameters
334 ----------
335 level : `str`
336 One of the python `logging` log level names.
338 Returns
339 -------
340 numericValue : `int`
341 The python `logging` numeric value for the log level.
342 """
343 if level == "VERBOSE":
344 return VERBOSE
345 elif level == "TRACE":
346 return TRACE
347 return getattr(logging, level, None)
349 @staticmethod
350 def _getLsstLogLevel(level: str) -> int | None:
351 """Get the numeric value for the given log level name.
353 If `lsst.log` is not setup this function will return `None` regardless
354 of input. `daf_butler` does not directly depend on `lsst.log` and so it
355 will not be setup when `daf_butler` is setup. Packages that depend on
356 `daf_butler` and use `lsst.log` may setup `lsst.log`.
358 Parameters
359 ----------
360 level : `str`
361 One of the python `logging` log level names.
363 Returns
364 -------
365 numericValue : `int` or `None`
366 The `lsst.log` numeric value.
368 Notes
369 -----
370 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger.
371 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be
372 converted to ``DEBUG``.
373 """
374 if lsstLog is None:
375 return None
376 if level == "VERBOSE":
377 level = "INFO"
378 elif level == "TRACE":
379 level = "DEBUG"
380 pylog_level = CliLog._getPyLogLevel(level)
381 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level)
383 class ComponentSettings:
384 """Container for log level values for a logging component."""
386 def __init__(self, component: str | None):
387 self.component = component
388 self.pythonLogLevel = logging.getLogger(component).level
389 self.lsstLogLevel = (
390 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None
391 )
392 if self.lsstLogLevel == -1:
393 self.lsstLogLevel = CliLog.defaultLsstLogLevel
395 def __repr__(self) -> str:
396 return (
397 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
398 f"lsstLogLevel={self.lsstLogLevel})"
399 )
401 @classmethod
402 def _recordComponentSetting(cls, component: str | None) -> None:
403 """Cache current levels for the given component in the list of
404 component levels.
405 """
406 componentSettings = cls.ComponentSettings(component)
407 cls._componentSettings.append(componentSettings)
409 @classmethod
410 def replayConfigState(cls, configState: list[tuple[Any, ...]]) -> None:
411 """Re-create configuration using configuration state recorded earlier.
413 Parameters
414 ----------
415 configState : `list` of `tuple`
416 Tuples contain a method as first item and arguments for the method,
417 in the same format as ``cls.configState``.
418 """
419 if cls._initialized or cls.configState:
420 # Already initialized, do not touch anything.
421 log = logging.getLogger(__name__)
422 log.warning("Log is already initialized, will not replay configuration.")
423 return
425 # execute each one in order
426 for call in configState:
427 method, *args = call
428 method(*args)