Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%
159 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "PrecisionLogFormatter",
25 "CliLog",
26)
28import datetime
29import logging
30import os
31from typing import Any
33try:
34 import lsst.log as lsstLog
35except ModuleNotFoundError:
36 lsstLog = None
38from lsst.utils.logging import TRACE, VERBOSE
40from ..core.logging import ButlerMDC, JsonLogFormatter
43class PrecisionLogFormatter(logging.Formatter):
44 """A log formatter that issues accurate timezone-aware timestamps."""
46 converter = datetime.datetime.fromtimestamp # type: ignore
48 use_local = True
49 """Control whether local time is displayed instead of UTC."""
51 def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str:
52 """Format the time as an aware datetime."""
53 ct: datetime.datetime = self.converter(record.created, tz=datetime.timezone.utc) # type: ignore
54 if self.use_local:
55 ct = ct.astimezone()
56 if datefmt:
57 s = ct.strftime(datefmt)
58 else:
59 s = ct.isoformat(sep="T", timespec="milliseconds")
60 return s
63class CliLog:
64 """Interface for managing python logging and ``lsst.log``.
66 This class defines log format strings for the log output and timestamp
67 formats. It also configures ``lsst.log`` to forward all messages to
68 Python `logging`.
70 This class can perform log uninitialization, which allows command line
71 interface code that initializes logging to run unit tests that execute in
72 batches, without affecting other unit tests. See ``resetLog``.
73 """
75 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
77 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
78 """The log format used when the lsst.log package is not importable and the
79 log is initialized with longlog=True."""
81 pylog_normalFmt = "{name} {levelname}: {message}"
82 """The log format used when the lsst.log package is not importable and the
83 log is initialized with longlog=False."""
85 configState: list[tuple[Any, ...]] = []
86 """Configuration state. Contains tuples where first item in a tuple is
87 a method and remaining items are arguments for the method.
88 """
90 _initialized = False
91 _componentSettings: list[ComponentSettings] = []
93 _fileHandlers: list[logging.FileHandler] = []
94 """Any FileHandler classes attached to the root logger by this class
95 that need to be closed on reset."""
97 @staticmethod
98 def root_loggers() -> set[str]:
99 """Return the default root logger.
101 Returns
102 -------
103 log_name : `set` of `str`
104 The name(s) of the root logger(s) to use when the log level is
105 being set without a log name being specified.
107 Notes
108 -----
109 The default is ``lsst`` (which controls the butler infrastructure)
110 but additional loggers can be specified by setting the environment
111 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain
112 multiple default loggers separated by a ``:``.
113 """
114 log_names = {"lsst"}
115 envvar = "DAF_BUTLER_ROOT_LOGGER"
116 if envvar in os.environ: 116 ↛ 117line 116 didn't jump to line 117, because the condition on line 116 was never true
117 log_names |= set(os.environ[envvar].split(":"))
118 return log_names
120 @classmethod
121 def initLog(
122 cls,
123 longlog: bool,
124 log_tty: bool = True,
125 log_file: tuple[str, ...] = (),
126 log_label: dict[str, str] | None = None,
127 ) -> None:
128 """Initialize logging. This should only be called once per program
129 execution. After the first call this will log a warning and return.
131 If lsst.log is importable, will add its log handler to the python
132 root logger's handlers.
134 Parameters
135 ----------
136 longlog : `bool`
137 If True, make log messages appear in long format, by default False.
138 log_tty : `bool`
139 Control whether a default stream handler is enabled that logs
140 to the terminal.
141 log_file : `tuple` of `str`
142 Path to files to write log records. If path ends in ``.json`` the
143 records will be written in JSON format. Else they will be written
144 in text format. If empty no log file will be created. Records
145 will be appended to this file if it exists.
146 log_label : `dict` of `str`
147 Keys and values to be stored in logging MDC for all JSON log
148 records. Keys will be upper-cased.
149 """
150 if cls._initialized:
151 # Unit tests that execute more than one command do end up
152 # calling this function multiple times in one program execution,
153 # so do log a debug but don't log an error or fail, just make the
154 # re-initialization a no-op.
155 log = logging.getLogger(__name__)
156 log.debug("Log is already initialized, returning without re-initializing.")
157 return
158 cls._initialized = True
159 cls._recordComponentSetting(None)
161 if lsstLog is not None:
162 # Ensure that log messages are forwarded back to python.
163 # Disable use of lsst.log MDC -- we expect butler uses to
164 # use ButlerMDC.
165 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
167 # Forward python lsst.log messages directly to python logging.
168 # This can bypass the C++ layer entirely but requires that
169 # MDC is set via ButlerMDC, rather than in lsst.log.
170 lsstLog.usePythonLogging()
172 formatter: logging.Formatter
173 if not log_tty:
174 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
175 elif longlog:
176 # Want to create our own Formatter so that we can get high
177 # precision timestamps. This requires we attach our own
178 # default stream handler.
179 defaultHandler = logging.StreamHandler()
180 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
181 defaultHandler.setFormatter(formatter)
183 logging.basicConfig(
184 level=logging.WARNING,
185 force=True,
186 handlers=[defaultHandler],
187 )
189 else:
190 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{")
192 # Initialize the root logger. Calling this ensures that both
193 # python loggers and lsst loggers are consistent in their default
194 # logging level.
195 cls._setLogLevel(".", "WARNING")
197 # Initialize default root logger level.
198 cls._setLogLevel(None, "INFO")
200 # also capture warnings and send them to logging
201 logging.captureWarnings(True)
203 # Create a record factory that ensures that an MDC is attached
204 # to the records. By default this is only used for long-log
205 # but always enable it for when someone adds a new handler
206 # that needs it.
207 ButlerMDC.add_mdc_log_record_factory()
209 # Set up the file logger
210 for file in log_file:
211 handler = logging.FileHandler(file)
212 if file.endswith(".json"):
213 formatter = JsonLogFormatter()
214 else:
215 if longlog:
216 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
217 else:
218 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
219 handler.setFormatter(formatter)
220 logging.getLogger().addHandler(handler)
221 cls._fileHandlers.append(handler)
223 # Add any requested MDC records.
224 if log_label:
225 for key, value in log_label.items():
226 ButlerMDC.MDC(key.upper(), value)
228 # remember this call
229 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
231 @classmethod
232 def resetLog(cls) -> None:
233 """Uninitialize the butler CLI Log handler and reset component log
234 levels.
236 If the lsst.log handler was added to the python root logger's handlers
237 in `initLog`, it will be removed here.
239 For each logger level that was set by this class, sets that logger's
240 level to the value it was before this class set it. For lsst.log, if a
241 component level was uninitialized, it will be set to
242 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
243 component back to an uninitialized state.
244 """
245 if lsstLog:
246 lsstLog.doNotUsePythonLogging()
247 for componentSetting in reversed(cls._componentSettings):
248 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
249 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
250 logger = logging.getLogger(componentSetting.component)
251 logger.setLevel(componentSetting.pythonLogLevel)
252 cls._setLogLevel(None, "INFO")
254 ButlerMDC.restore_log_record_factory()
256 # Remove the FileHandler we may have attached.
257 root = logging.getLogger()
258 for handler in cls._fileHandlers:
259 handler.close()
260 root.removeHandler(handler)
262 cls._fileHandlers.clear()
263 cls._initialized = False
264 cls.configState = []
266 @classmethod
267 def setLogLevels(cls, logLevels: list[tuple[str | None, str]] | dict[str, str]) -> None:
268 """Set log level for one or more components or the root logger.
270 Parameters
271 ----------
272 logLevels : `list` of `tuple`
273 per-component logging levels, each item in the list is a tuple
274 (component, level), `component` is a logger name or an empty string
275 or `None` for default root logger, `level` is a logging level name,
276 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
278 Notes
279 -----
280 The special name ``.`` can be used to set the Python root
281 logger.
282 """
283 if isinstance(logLevels, dict):
284 logLevels = list(logLevels.items())
286 # configure individual loggers
287 for component, level in logLevels:
288 cls._setLogLevel(component, level)
289 # remember this call
290 cls.configState.append((cls._setLogLevel, component, level))
292 @classmethod
293 def _setLogLevel(cls, component: str | None, level: str) -> None:
294 """Set the log level for the given component. Record the current log
295 level of the component so that it can be restored when resetting this
296 log.
298 Parameters
299 ----------
300 component : `str` or None
301 The name of the log component or None for the default logger.
302 The root logger can be specified either by an empty string or
303 with the special name ``.``.
304 level : `str`
305 A valid python logging level.
306 """
307 components: set[str | None]
308 if component is None:
309 components = {comp for comp in cls.root_loggers()}
310 elif not component or component == ".":
311 components = {None}
312 else:
313 components = {component}
314 for component in components:
315 cls._recordComponentSetting(component)
316 if lsstLog is not None:
317 lsstLogger = lsstLog.Log.getLogger(component or "")
318 lsstLogger.setLevel(cls._getLsstLogLevel(level))
319 pylevel = cls._getPyLogLevel(level)
320 if pylevel is not None:
321 logging.getLogger(component or None).setLevel(pylevel)
323 @staticmethod
324 def _getPyLogLevel(level: str) -> int | None:
325 """Get the numeric value for the given log level name.
327 Parameters
328 ----------
329 level : `str`
330 One of the python `logging` log level names.
332 Returns
333 -------
334 numericValue : `int`
335 The python `logging` numeric value for the log level.
336 """
337 if level == "VERBOSE":
338 return VERBOSE
339 elif level == "TRACE":
340 return TRACE
341 return getattr(logging, level, None)
343 @staticmethod
344 def _getLsstLogLevel(level: str) -> int | None:
345 """Get the numeric value for the given log level name.
347 If `lsst.log` is not setup this function will return `None` regardless
348 of input. `daf_butler` does not directly depend on `lsst.log` and so it
349 will not be setup when `daf_butler` is setup. Packages that depend on
350 `daf_butler` and use `lsst.log` may setup `lsst.log`.
352 Parameters
353 ----------
354 level : `str`
355 One of the python `logging` log level names.
357 Returns
358 -------
359 numericValue : `int` or `None`
360 The `lsst.log` numeric value.
362 Notes
363 -----
364 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger.
365 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be
366 converted to ``DEBUG``.
367 """
368 if lsstLog is None:
369 return None
370 if level == "VERBOSE":
371 level = "INFO"
372 elif level == "TRACE":
373 level = "DEBUG"
374 pylog_level = CliLog._getPyLogLevel(level)
375 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level)
377 class ComponentSettings:
378 """Container for log level values for a logging component."""
380 def __init__(self, component: str | None):
381 self.component = component
382 self.pythonLogLevel = logging.getLogger(component).level
383 self.lsstLogLevel = (
384 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None
385 )
386 if self.lsstLogLevel == -1:
387 self.lsstLogLevel = CliLog.defaultLsstLogLevel
389 def __repr__(self) -> str:
390 return (
391 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
392 f"lsstLogLevel={self.lsstLogLevel})"
393 )
395 @classmethod
396 def _recordComponentSetting(cls, component: str | None) -> None:
397 """Cache current levels for the given component in the list of
398 component levels.
399 """
400 componentSettings = cls.ComponentSettings(component)
401 cls._componentSettings.append(componentSettings)
403 @classmethod
404 def replayConfigState(cls, configState: list[tuple[Any, ...]]) -> None:
405 """Re-create configuration using configuration state recorded earlier.
407 Parameters
408 ----------
409 configState : `list` of `tuple`
410 Tuples contain a method as first item and arguments for the method,
411 in the same format as ``cls.configState``.
412 """
413 if cls._initialized or cls.configState:
414 # Already initialized, do not touch anything.
415 log = logging.getLogger(__name__)
416 log.warning("Log is already initialized, will not replay configuration.")
417 return
419 # execute each one in order
420 for call in configState:
421 method, *args = call
422 method(*args)