Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
24import os
25from typing import Dict, Optional, Set, Tuple
27try:
28 import lsst.log as lsstLog
29except ModuleNotFoundError:
30 lsstLog = None
32from lsst.utils.logging import TRACE, VERBOSE
34from ..core.logging import ButlerMDC, JsonLogFormatter
37class PrecisionLogFormatter(logging.Formatter):
38 """A log formatter that issues accurate timezone-aware timestamps."""
40 converter = datetime.datetime.fromtimestamp
42 use_local = True
43 """Control whether local time is displayed instead of UTC."""
45 def formatTime(self, record, datefmt=None):
46 """Format the time as an aware datetime."""
47 ct = self.converter(record.created, tz=datetime.timezone.utc)
48 if self.use_local:
49 ct = ct.astimezone()
50 if datefmt:
51 s = ct.strftime(datefmt)
52 else:
53 s = ct.isoformat(sep="T", timespec="milliseconds")
54 return s
57class CliLog:
58 """Interface for managing python logging and ``lsst.log``.
60 This class defines log format strings for the log output and timestamp
61 formats. It also configures ``lsst.log`` to forward all messages to
62 Python `logging`.
64 This class can perform log uninitialization, which allows command line
65 interface code that initializes logging to run unit tests that execute in
66 batches, without affecting other unit tests. See ``resetLog``."""
68 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
70 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
71 """The log format used when the lsst.log package is not importable and the
72 log is initialized with longlog=True."""
74 pylog_normalFmt = "{name} {levelname}: {message}"
75 """The log format used when the lsst.log package is not importable and the
76 log is initialized with longlog=False."""
78 configState = []
79 """Configuration state. Contains tuples where first item in a tuple is
80 a method and remaining items are arguments for the method.
81 """
83 _initialized = False
84 _componentSettings = []
86 _fileHandlers = []
87 """Any FileHandler classes attached to the root logger by this class
88 that need to be closed on reset."""
90 @staticmethod
91 def root_loggers() -> Set[str]:
92 """Return the default root logger.
94 Returns
95 -------
96 log_name : `set` of `str`
97 The name(s) of the root logger(s) to use when the log level is
98 being set without a log name being specified.
100 Notes
101 -----
102 The default is ``lsst`` (which controls the butler infrastructure)
103 but additional loggers can be specified by setting the environment
104 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain
105 multiple default loggers separated by a ``:``.
106 """
107 log_names = set(["lsst"])
108 envvar = "DAF_BUTLER_ROOT_LOGGER"
109 if envvar in os.environ: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true
110 log_names |= set(os.environ[envvar].split(":"))
111 return log_names
113 @classmethod
114 def initLog(
115 cls,
116 longlog: bool,
117 log_tty: bool = True,
118 log_file: Tuple[str, ...] = (),
119 log_label: Optional[Dict[str, str]] = None,
120 ):
121 """Initialize logging. This should only be called once per program
122 execution. After the first call this will log a warning and return.
124 If lsst.log is importable, will add its log handler to the python
125 root logger's handlers.
127 Parameters
128 ----------
129 longlog : `bool`
130 If True, make log messages appear in long format, by default False.
131 log_tty : `bool`
132 Control whether a default stream handler is enabled that logs
133 to the terminal.
134 log_file : `tuple` of `str`
135 Path to files to write log records. If path ends in ``.json`` the
136 records will be written in JSON format. Else they will be written
137 in text format. If empty no log file will be created. Records
138 will be appended to this file if it exists.
139 log_label : `dict` of `str`
140 Keys and values to be stored in logging MDC for all JSON log
141 records. Keys will be upper-cased.
142 """
143 if cls._initialized:
144 # Unit tests that execute more than one command do end up
145 # calling this function multiple times in one program execution,
146 # so do log a debug but don't log an error or fail, just make the
147 # re-initialization a no-op.
148 log = logging.getLogger(__name__)
149 log.debug("Log is already initialized, returning without re-initializing.")
150 return
151 cls._initialized = True
152 cls._recordComponentSetting(None)
154 if lsstLog is not None:
155 # Ensure that log messages are forwarded back to python.
156 # Disable use of lsst.log MDC -- we expect butler uses to
157 # use ButlerMDC.
158 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
160 # Forward python lsst.log messages directly to python logging.
161 # This can bypass the C++ layer entirely but requires that
162 # MDC is set via ButlerMDC, rather than in lsst.log.
163 lsstLog.usePythonLogging()
165 if not log_tty:
166 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
167 elif longlog:
169 # Want to create our own Formatter so that we can get high
170 # precision timestamps. This requires we attach our own
171 # default stream handler.
172 defaultHandler = logging.StreamHandler()
173 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
174 defaultHandler.setFormatter(formatter)
176 logging.basicConfig(
177 level=logging.WARNING,
178 force=True,
179 handlers=[defaultHandler],
180 )
182 else:
183 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{")
185 # Initialize root logger level.
186 cls._setLogLevel(None, "INFO")
188 # also capture warnings and send them to logging
189 logging.captureWarnings(True)
191 # Create a record factory that ensures that an MDC is attached
192 # to the records. By default this is only used for long-log
193 # but always enable it for when someone adds a new handler
194 # that needs it.
195 ButlerMDC.add_mdc_log_record_factory()
197 # Set up the file logger
198 for file in log_file:
199 handler = logging.FileHandler(file)
200 if file.endswith(".json"):
201 formatter = JsonLogFormatter()
202 else:
203 if longlog:
204 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
205 else:
206 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
207 handler.setFormatter(formatter)
208 logging.getLogger().addHandler(handler)
209 cls._fileHandlers.append(handler)
211 # Add any requested MDC records.
212 if log_label:
213 for key, value in log_label.items():
214 ButlerMDC.MDC(key.upper(), value)
216 # remember this call
217 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
219 @classmethod
220 def resetLog(cls):
221 """Uninitialize the butler CLI Log handler and reset component log
222 levels.
224 If the lsst.log handler was added to the python root logger's handlers
225 in `initLog`, it will be removed here.
227 For each logger level that was set by this class, sets that logger's
228 level to the value it was before this class set it. For lsst.log, if a
229 component level was uninitialized, it will be set to
230 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
231 component back to an uninitialized state.
232 """
233 if lsstLog:
234 lsstLog.doNotUsePythonLogging()
235 for componentSetting in reversed(cls._componentSettings):
236 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
237 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
238 logger = logging.getLogger(componentSetting.component)
239 logger.setLevel(componentSetting.pythonLogLevel)
240 cls._setLogLevel(None, "INFO")
242 ButlerMDC.restore_log_record_factory()
244 # Remove the FileHandler we may have attached.
245 root = logging.getLogger()
246 for handler in cls._fileHandlers:
247 handler.close()
248 root.removeHandler(handler)
250 cls._fileHandlers.clear()
251 cls._initialized = False
252 cls.configState = []
254 @classmethod
255 def setLogLevels(cls, logLevels):
256 """Set log level for one or more components or the root logger.
258 Parameters
259 ----------
260 logLevels : `list` of `tuple`
261 per-component logging levels, each item in the list is a tuple
262 (component, level), `component` is a logger name or an empty string
263 or `None` for default root logger, `level` is a logging level name,
264 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
266 Notes
267 -----
268 The special name ``.`` can be used to set the Python root
269 logger.
270 """
271 if isinstance(logLevels, dict):
272 logLevels = logLevels.items()
274 # configure individual loggers
275 for component, level in logLevels:
276 cls._setLogLevel(component, level)
277 # remember this call
278 cls.configState.append((cls._setLogLevel, component, level))
280 @classmethod
281 def _setLogLevel(cls, component, level):
282 """Set the log level for the given component. Record the current log
283 level of the component so that it can be restored when resetting this
284 log.
286 Parameters
287 ----------
288 component : `str` or None
289 The name of the log component or None for the default logger.
290 The root logger can be specified either by an empty string or
291 with the special name ``.``.
292 level : `str`
293 A valid python logging level.
294 """
295 components: Set[Optional[str]]
296 if component is None:
297 components = cls.root_loggers()
298 elif not component or component == ".":
299 components = {None}
300 else:
301 components = {component}
302 for component in components:
303 cls._recordComponentSetting(component)
304 if lsstLog is not None:
305 lsstLogger = lsstLog.Log.getLogger(component or "")
306 lsstLogger.setLevel(cls._getLsstLogLevel(level))
307 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
309 @staticmethod
310 def _getPyLogLevel(level):
311 """Get the numeric value for the given log level name.
313 Parameters
314 ----------
315 level : `str`
316 One of the python `logging` log level names.
318 Returns
319 -------
320 numericValue : `int`
321 The python `logging` numeric value for the log level.
322 """
323 if level == "VERBOSE":
324 return VERBOSE
325 elif level == "TRACE":
326 return TRACE
327 return getattr(logging, level, None)
329 @staticmethod
330 def _getLsstLogLevel(level):
331 """Get the numeric value for the given log level name.
333 If `lsst.log` is not setup this function will return `None` regardless
334 of input. `daf_butler` does not directly depend on `lsst.log` and so it
335 will not be setup when `daf_butler` is setup. Packages that depend on
336 `daf_butler` and use `lsst.log` may setup `lsst.log`.
338 Parameters
339 ----------
340 level : `str`
341 One of the python `logging` log level names.
343 Returns
344 -------
345 numericValue : `int` or `None`
346 The `lsst.log` numeric value.
348 Notes
349 -----
350 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger.
351 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be
352 converted to ``DEBUG``.
353 """
354 if lsstLog is None:
355 return None
356 if level == "VERBOSE":
357 level = "INFO"
358 elif level == "TRACE":
359 level = "DEBUG"
360 pylog_level = CliLog._getPyLogLevel(level)
361 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level)
363 class ComponentSettings:
364 """Container for log level values for a logging component."""
366 def __init__(self, component):
367 self.component = component
368 self.pythonLogLevel = logging.getLogger(component).level
369 self.lsstLogLevel = (
370 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None
371 )
372 if self.lsstLogLevel == -1:
373 self.lsstLogLevel = CliLog.defaultLsstLogLevel
375 def __repr__(self):
376 return (
377 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
378 f"lsstLogLevel={self.lsstLogLevel})"
379 )
381 @classmethod
382 def _recordComponentSetting(cls, component):
383 """Cache current levels for the given component in the list of
384 component levels."""
385 componentSettings = cls.ComponentSettings(component)
386 cls._componentSettings.append(componentSettings)
388 @classmethod
389 def replayConfigState(cls, configState):
390 """Re-create configuration using configuration state recorded earlier.
392 Parameters
393 ----------
394 configState : `list` of `tuple`
395 Tuples contain a method as first item and arguments for the method,
396 in the same format as ``cls.configState``.
397 """
398 if cls._initialized or cls.configState:
399 # Already initialized, do not touch anything.
400 log = logging.getLogger(__name__)
401 log.warning("Log is already initialized, will not replay configuration.")
402 return
404 # execute each one in order
405 for call in configState:
406 method, *args = call
407 method(*args)