Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
24from typing import Dict, Optional, Tuple
26try:
27 import lsst.log as lsstLog
28except ModuleNotFoundError:
29 lsstLog = None
31from lsst.utils.logging import VERBOSE
33from ..core.logging import ButlerMDC, JsonLogFormatter
36class PrecisionLogFormatter(logging.Formatter):
37 """A log formatter that issues accurate timezone-aware timestamps."""
39 converter = datetime.datetime.fromtimestamp
41 use_local = True
42 """Control whether local time is displayed instead of UTC."""
44 def formatTime(self, record, datefmt=None):
45 """Format the time as an aware datetime."""
46 ct = self.converter(record.created, tz=datetime.timezone.utc)
47 if self.use_local:
48 ct = ct.astimezone()
49 if datefmt:
50 s = ct.strftime(datefmt)
51 else:
52 s = ct.isoformat(sep="T", timespec="milliseconds")
53 return s
56class CliLog:
57 """Interface for managing python logging and ``lsst.log``.
59 This class defines log format strings for the log output and timestamp
60 formats. It also configures ``lsst.log`` to forward all messages to
61 Python `logging`.
63 This class can perform log uninitialization, which allows command line
64 interface code that initializes logging to run unit tests that execute in
65 batches, without affecting other unit tests. See ``resetLog``."""
67 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
69 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
70 """The log format used when the lsst.log package is not importable and the
71 log is initialized with longlog=True."""
73 pylog_normalFmt = "{name} {levelname}: {message}"
74 """The log format used when the lsst.log package is not importable and the
75 log is initialized with longlog=False."""
77 configState = []
78 """Configuration state. Contains tuples where first item in a tuple is
79 a method and remaining items are arguments for the method.
80 """
82 _initialized = False
83 _componentSettings = []
85 _fileHandlers = []
86 """Any FileHandler classes attached to the root logger by this class
87 that need to be closed on reset."""
89 @classmethod
90 def initLog(
91 cls,
92 longlog: bool,
93 log_tty: bool = True,
94 log_file: Tuple[str, ...] = (),
95 log_label: Optional[Dict[str, str]] = None,
96 ):
97 """Initialize logging. This should only be called once per program
98 execution. After the first call this will log a warning and return.
100 If lsst.log is importable, will add its log handler to the python
101 root logger's handlers.
103 Parameters
104 ----------
105 longlog : `bool`
106 If True, make log messages appear in long format, by default False.
107 log_tty : `bool`
108 Control whether a default stream handler is enabled that logs
109 to the terminal.
110 log_file : `tuple` of `str`
111 Path to files to write log records. If path ends in ``.json`` the
112 records will be written in JSON format. Else they will be written
113 in text format. If empty no log file will be created. Records
114 will be appended to this file if it exists.
115 log_label : `dict` of `str`
116 Keys and values to be stored in logging MDC for all JSON log
117 records. Keys will be upper-cased.
118 """
119 if cls._initialized:
120 # Unit tests that execute more than one command do end up
121 # calling this function multiple times in one program execution,
122 # so do log a debug but don't log an error or fail, just make the
123 # re-initialization a no-op.
124 log = logging.getLogger(__name__)
125 log.debug("Log is already initialized, returning without re-initializing.")
126 return
127 cls._initialized = True
128 cls._recordComponentSetting(None)
130 if lsstLog is not None:
131 # Ensure that log messages are forwarded back to python.
132 # Disable use of lsst.log MDC -- we expect butler uses to
133 # use ButlerMDC.
134 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
136 # Forward python lsst.log messages directly to python logging.
137 # This can bypass the C++ layer entirely but requires that
138 # MDC is set via ButlerMDC, rather than in lsst.log.
139 lsstLog.usePythonLogging()
141 if not log_tty:
142 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
143 elif longlog:
145 # Want to create our own Formatter so that we can get high
146 # precision timestamps. This requires we attach our own
147 # default stream handler.
148 defaultHandler = logging.StreamHandler()
149 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
150 defaultHandler.setFormatter(formatter)
152 logging.basicConfig(
153 level=logging.INFO,
154 force=True,
155 handlers=[defaultHandler],
156 )
158 else:
159 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{")
161 # Initialize root logger level.
162 cls._setLogLevel(None, "INFO")
164 # also capture warnings and send them to logging
165 logging.captureWarnings(True)
167 # Create a record factory that ensures that an MDC is attached
168 # to the records. By default this is only used for long-log
169 # but always enable it for when someone adds a new handler
170 # that needs it.
171 ButlerMDC.add_mdc_log_record_factory()
173 # Set up the file logger
174 for file in log_file:
175 handler = logging.FileHandler(file)
176 if file.endswith(".json"):
177 formatter = JsonLogFormatter()
178 else:
179 if longlog:
180 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
181 else:
182 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
183 handler.setFormatter(formatter)
184 logging.getLogger().addHandler(handler)
185 cls._fileHandlers.append(handler)
187 # Add any requested MDC records.
188 if log_label:
189 for key, value in log_label.items():
190 ButlerMDC.MDC(key.upper(), value)
192 # remember this call
193 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
195 @classmethod
196 def resetLog(cls):
197 """Uninitialize the butler CLI Log handler and reset component log
198 levels.
200 If the lsst.log handler was added to the python root logger's handlers
201 in `initLog`, it will be removed here.
203 For each logger level that was set by this class, sets that logger's
204 level to the value it was before this class set it. For lsst.log, if a
205 component level was uninitialized, it will be set to
206 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
207 component back to an uninitialized state.
208 """
209 if lsstLog:
210 lsstLog.doNotUsePythonLogging()
211 for componentSetting in reversed(cls._componentSettings):
212 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
213 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
214 logger = logging.getLogger(componentSetting.component)
215 logger.setLevel(componentSetting.pythonLogLevel)
216 cls._setLogLevel(None, "INFO")
218 ButlerMDC.restore_log_record_factory()
220 # Remove the FileHandler we may have attached.
221 root = logging.getLogger()
222 for handler in cls._fileHandlers:
223 handler.close()
224 root.removeHandler(handler)
226 cls._fileHandlers.clear()
227 cls._initialized = False
228 cls.configState = []
230 @classmethod
231 def setLogLevels(cls, logLevels):
232 """Set log level for one or more components or the root logger.
234 Parameters
235 ----------
236 logLevels : `list` of `tuple`
237 per-component logging levels, each item in the list is a tuple
238 (component, level), `component` is a logger name or an empty string
239 or `None` for root logger, `level` is a logging level name, one of
240 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
241 """
242 if isinstance(logLevels, dict):
243 logLevels = logLevels.items()
245 # configure individual loggers
246 for component, level in logLevels:
247 cls._setLogLevel(component, level)
248 # remember this call
249 cls.configState.append((cls._setLogLevel, component, level))
251 @classmethod
252 def _setLogLevel(cls, component, level):
253 """Set the log level for the given component. Record the current log
254 level of the component so that it can be restored when resetting this
255 log.
257 Parameters
258 ----------
259 component : `str` or None
260 The name of the log component or None for the root logger.
261 level : `str`
262 A valid python logging level.
263 """
264 cls._recordComponentSetting(component)
265 if lsstLog is not None:
266 lsstLogger = lsstLog.Log.getLogger(component or "")
267 lsstLogger.setLevel(cls._getLsstLogLevel(level))
268 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
270 @staticmethod
271 def _getPyLogLevel(level):
272 """Get the numeric value for the given log level name.
274 Parameters
275 ----------
276 level : `str`
277 One of the python `logging` log level names.
279 Returns
280 -------
281 numericValue : `int`
282 The python `logging` numeric value for the log level.
283 """
284 if level == "VERBOSE":
285 return VERBOSE
286 return getattr(logging, level, None)
288 @staticmethod
289 def _getLsstLogLevel(level):
290 """Get the numeric value for the given log level name.
292 If `lsst.log` is not setup this function will return `None` regardless
293 of input. `daf_butler` does not directly depend on `lsst.log` and so it
294 will not be setup when `daf_butler` is setup. Packages that depend on
295 `daf_butler` and use `lsst.log` may setup `lsst.log`.
297 Parameters
298 ----------
299 level : `str`
300 One of the python `logging` log level names.
302 Returns
303 -------
304 numericValue : `int` or `None`
305 The `lsst.log` numeric value.
307 Notes
308 -----
309 ``VERBOSE`` logging is not supported by the LSST logger and so will
310 always be converted to ``INFO``.
311 """
312 if lsstLog is None:
313 return None
314 if level == "VERBOSE":
315 level = "INFO"
316 pylog_level = CliLog._getPyLogLevel(level)
317 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level)
319 class ComponentSettings:
320 """Container for log level values for a logging component."""
322 def __init__(self, component):
323 self.component = component
324 self.pythonLogLevel = logging.getLogger(component).level
325 self.lsstLogLevel = (
326 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None
327 )
328 if self.lsstLogLevel == -1:
329 self.lsstLogLevel = CliLog.defaultLsstLogLevel
331 def __repr__(self):
332 return (
333 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
334 f"lsstLogLevel={self.lsstLogLevel})"
335 )
337 @classmethod
338 def _recordComponentSetting(cls, component):
339 """Cache current levels for the given component in the list of
340 component levels."""
341 componentSettings = cls.ComponentSettings(component)
342 cls._componentSettings.append(componentSettings)
344 @classmethod
345 def replayConfigState(cls, configState):
346 """Re-create configuration using configuration state recorded earlier.
348 Parameters
349 ----------
350 configState : `list` of `tuple`
351 Tuples contain a method as first item and arguments for the method,
352 in the same format as ``cls.configState``.
353 """
354 if cls._initialized or cls.configState:
355 # Already initialized, do not touch anything.
356 log = logging.getLogger(__name__)
357 log.warning("Log is already initialized, will not replay configuration.")
358 return
360 # execute each one in order
361 for call in configState:
362 method, *args = call
363 method(*args)