Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
24import os
25from typing import Dict, Optional, Set, Tuple
27try:
28 import lsst.log as lsstLog
29except ModuleNotFoundError:
30 lsstLog = None
32from lsst.utils.logging import TRACE, VERBOSE
34from ..core.logging import ButlerMDC, JsonLogFormatter
37class PrecisionLogFormatter(logging.Formatter):
38 """A log formatter that issues accurate timezone-aware timestamps."""
40 converter = datetime.datetime.fromtimestamp
42 use_local = True
43 """Control whether local time is displayed instead of UTC."""
45 def formatTime(self, record, datefmt=None):
46 """Format the time as an aware datetime."""
47 ct = self.converter(record.created, tz=datetime.timezone.utc)
48 if self.use_local:
49 ct = ct.astimezone()
50 if datefmt:
51 s = ct.strftime(datefmt)
52 else:
53 s = ct.isoformat(sep="T", timespec="milliseconds")
54 return s
57class CliLog:
58 """Interface for managing python logging and ``lsst.log``.
60 This class defines log format strings for the log output and timestamp
61 formats. It also configures ``lsst.log`` to forward all messages to
62 Python `logging`.
64 This class can perform log uninitialization, which allows command line
65 interface code that initializes logging to run unit tests that execute in
66 batches, without affecting other unit tests. See ``resetLog``."""
68 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
70 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
71 """The log format used when the lsst.log package is not importable and the
72 log is initialized with longlog=True."""
74 pylog_normalFmt = "{name} {levelname}: {message}"
75 """The log format used when the lsst.log package is not importable and the
76 log is initialized with longlog=False."""
78 configState = []
79 """Configuration state. Contains tuples where first item in a tuple is
80 a method and remaining items are arguments for the method.
81 """
83 _initialized = False
84 _componentSettings = []
86 _fileHandlers = []
87 """Any FileHandler classes attached to the root logger by this class
88 that need to be closed on reset."""
90 @staticmethod
91 def root_loggers() -> Set[str]:
92 """Return the default root logger.
94 Returns
95 -------
96 log_name : `set` of `str`
97 The name(s) of the root logger(s) to use when the log level is
98 being set without a log name being specified.
100 Notes
101 -----
102 The default is ``lsst`` (which controls the butler infrastructure)
103 but additional loggers can be specified by setting the environment
104 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain
105 multiple default loggers separated by a ``:``.
106 """
107 log_names = set(["lsst"])
108 envvar = "DAF_BUTLER_ROOT_LOGGER"
109 if envvar in os.environ: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true
110 log_names |= set(os.environ[envvar].split(":"))
111 return log_names
113 @classmethod
114 def initLog(
115 cls,
116 longlog: bool,
117 log_tty: bool = True,
118 log_file: Tuple[str, ...] = (),
119 log_label: Optional[Dict[str, str]] = None,
120 ):
121 """Initialize logging. This should only be called once per program
122 execution. After the first call this will log a warning and return.
124 If lsst.log is importable, will add its log handler to the python
125 root logger's handlers.
127 Parameters
128 ----------
129 longlog : `bool`
130 If True, make log messages appear in long format, by default False.
131 log_tty : `bool`
132 Control whether a default stream handler is enabled that logs
133 to the terminal.
134 log_file : `tuple` of `str`
135 Path to files to write log records. If path ends in ``.json`` the
136 records will be written in JSON format. Else they will be written
137 in text format. If empty no log file will be created. Records
138 will be appended to this file if it exists.
139 log_label : `dict` of `str`
140 Keys and values to be stored in logging MDC for all JSON log
141 records. Keys will be upper-cased.
142 """
143 if cls._initialized:
144 # Unit tests that execute more than one command do end up
145 # calling this function multiple times in one program execution,
146 # so do log a debug but don't log an error or fail, just make the
147 # re-initialization a no-op.
148 log = logging.getLogger(__name__)
149 log.debug("Log is already initialized, returning without re-initializing.")
150 return
151 cls._initialized = True
152 cls._recordComponentSetting(None)
154 if lsstLog is not None:
155 # Ensure that log messages are forwarded back to python.
156 # Disable use of lsst.log MDC -- we expect butler uses to
157 # use ButlerMDC.
158 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
160 # Forward python lsst.log messages directly to python logging.
161 # This can bypass the C++ layer entirely but requires that
162 # MDC is set via ButlerMDC, rather than in lsst.log.
163 lsstLog.usePythonLogging()
165 if not log_tty:
166 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
167 elif longlog:
169 # Want to create our own Formatter so that we can get high
170 # precision timestamps. This requires we attach our own
171 # default stream handler.
172 defaultHandler = logging.StreamHandler()
173 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
174 defaultHandler.setFormatter(formatter)
176 logging.basicConfig(
177 level=logging.WARNING,
178 force=True,
179 handlers=[defaultHandler],
180 )
182 else:
183 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{")
185 # Initialize the root logger. Calling this ensures that both
186 # python loggers and lsst loggers are consistent in their default
187 # logging level.
188 cls._setLogLevel(".", "WARNING")
190 # Initialize default root logger level.
191 cls._setLogLevel(None, "INFO")
193 # also capture warnings and send them to logging
194 logging.captureWarnings(True)
196 # Create a record factory that ensures that an MDC is attached
197 # to the records. By default this is only used for long-log
198 # but always enable it for when someone adds a new handler
199 # that needs it.
200 ButlerMDC.add_mdc_log_record_factory()
202 # Set up the file logger
203 for file in log_file:
204 handler = logging.FileHandler(file)
205 if file.endswith(".json"):
206 formatter = JsonLogFormatter()
207 else:
208 if longlog:
209 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
210 else:
211 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
212 handler.setFormatter(formatter)
213 logging.getLogger().addHandler(handler)
214 cls._fileHandlers.append(handler)
216 # Add any requested MDC records.
217 if log_label:
218 for key, value in log_label.items():
219 ButlerMDC.MDC(key.upper(), value)
221 # remember this call
222 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
224 @classmethod
225 def resetLog(cls):
226 """Uninitialize the butler CLI Log handler and reset component log
227 levels.
229 If the lsst.log handler was added to the python root logger's handlers
230 in `initLog`, it will be removed here.
232 For each logger level that was set by this class, sets that logger's
233 level to the value it was before this class set it. For lsst.log, if a
234 component level was uninitialized, it will be set to
235 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
236 component back to an uninitialized state.
237 """
238 if lsstLog:
239 lsstLog.doNotUsePythonLogging()
240 for componentSetting in reversed(cls._componentSettings):
241 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
242 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
243 logger = logging.getLogger(componentSetting.component)
244 logger.setLevel(componentSetting.pythonLogLevel)
245 cls._setLogLevel(None, "INFO")
247 ButlerMDC.restore_log_record_factory()
249 # Remove the FileHandler we may have attached.
250 root = logging.getLogger()
251 for handler in cls._fileHandlers:
252 handler.close()
253 root.removeHandler(handler)
255 cls._fileHandlers.clear()
256 cls._initialized = False
257 cls.configState = []
259 @classmethod
260 def setLogLevels(cls, logLevels):
261 """Set log level for one or more components or the root logger.
263 Parameters
264 ----------
265 logLevels : `list` of `tuple`
266 per-component logging levels, each item in the list is a tuple
267 (component, level), `component` is a logger name or an empty string
268 or `None` for default root logger, `level` is a logging level name,
269 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
271 Notes
272 -----
273 The special name ``.`` can be used to set the Python root
274 logger.
275 """
276 if isinstance(logLevels, dict):
277 logLevels = logLevels.items()
279 # configure individual loggers
280 for component, level in logLevels:
281 cls._setLogLevel(component, level)
282 # remember this call
283 cls.configState.append((cls._setLogLevel, component, level))
285 @classmethod
286 def _setLogLevel(cls, component, level):
287 """Set the log level for the given component. Record the current log
288 level of the component so that it can be restored when resetting this
289 log.
291 Parameters
292 ----------
293 component : `str` or None
294 The name of the log component or None for the default logger.
295 The root logger can be specified either by an empty string or
296 with the special name ``.``.
297 level : `str`
298 A valid python logging level.
299 """
300 components: Set[Optional[str]]
301 if component is None:
302 components = cls.root_loggers()
303 elif not component or component == ".":
304 components = {None}
305 else:
306 components = {component}
307 for component in components:
308 cls._recordComponentSetting(component)
309 if lsstLog is not None:
310 lsstLogger = lsstLog.Log.getLogger(component or "")
311 lsstLogger.setLevel(cls._getLsstLogLevel(level))
312 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
314 @staticmethod
315 def _getPyLogLevel(level):
316 """Get the numeric value for the given log level name.
318 Parameters
319 ----------
320 level : `str`
321 One of the python `logging` log level names.
323 Returns
324 -------
325 numericValue : `int`
326 The python `logging` numeric value for the log level.
327 """
328 if level == "VERBOSE":
329 return VERBOSE
330 elif level == "TRACE":
331 return TRACE
332 return getattr(logging, level, None)
334 @staticmethod
335 def _getLsstLogLevel(level):
336 """Get the numeric value for the given log level name.
338 If `lsst.log` is not setup this function will return `None` regardless
339 of input. `daf_butler` does not directly depend on `lsst.log` and so it
340 will not be setup when `daf_butler` is setup. Packages that depend on
341 `daf_butler` and use `lsst.log` may setup `lsst.log`.
343 Parameters
344 ----------
345 level : `str`
346 One of the python `logging` log level names.
348 Returns
349 -------
350 numericValue : `int` or `None`
351 The `lsst.log` numeric value.
353 Notes
354 -----
355 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger.
356 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be
357 converted to ``DEBUG``.
358 """
359 if lsstLog is None:
360 return None
361 if level == "VERBOSE":
362 level = "INFO"
363 elif level == "TRACE":
364 level = "DEBUG"
365 pylog_level = CliLog._getPyLogLevel(level)
366 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level)
368 class ComponentSettings:
369 """Container for log level values for a logging component."""
371 def __init__(self, component):
372 self.component = component
373 self.pythonLogLevel = logging.getLogger(component).level
374 self.lsstLogLevel = (
375 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None
376 )
377 if self.lsstLogLevel == -1:
378 self.lsstLogLevel = CliLog.defaultLsstLogLevel
380 def __repr__(self):
381 return (
382 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
383 f"lsstLogLevel={self.lsstLogLevel})"
384 )
386 @classmethod
387 def _recordComponentSetting(cls, component):
388 """Cache current levels for the given component in the list of
389 component levels."""
390 componentSettings = cls.ComponentSettings(component)
391 cls._componentSettings.append(componentSettings)
393 @classmethod
394 def replayConfigState(cls, configState):
395 """Re-create configuration using configuration state recorded earlier.
397 Parameters
398 ----------
399 configState : `list` of `tuple`
400 Tuples contain a method as first item and arguments for the method,
401 in the same format as ``cls.configState``.
402 """
403 if cls._initialized or cls.configState:
404 # Already initialized, do not touch anything.
405 log = logging.getLogger(__name__)
406 log.warning("Log is already initialized, will not replay configuration.")
407 return
409 # execute each one in order
410 for call in configState:
411 method, *args = call
412 method(*args)