Coverage for python/lsst/daf/butler/cli/cliLog.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
24from typing import Tuple, Optional, Dict
26try:
27 import lsst.log as lsstLog
28except ModuleNotFoundError:
29 lsstLog = None
31from lsst.utils.logging import VERBOSE
32from ..core.logging import JsonLogFormatter, ButlerMDC
35class PrecisionLogFormatter(logging.Formatter):
36 """A log formatter that issues accurate timezone-aware timestamps."""
38 converter = datetime.datetime.fromtimestamp
40 use_local = True
41 """Control whether local time is displayed instead of UTC."""
43 def formatTime(self, record, datefmt=None):
44 """Format the time as an aware datetime."""
45 ct = self.converter(record.created, tz=datetime.timezone.utc)
46 if self.use_local:
47 ct = ct.astimezone()
48 if datefmt:
49 s = ct.strftime(datefmt)
50 else:
51 s = ct.isoformat(sep='T', timespec='milliseconds')
52 return s
55class CliLog:
56 """Interface for managing python logging and ``lsst.log``.
58 This class defines log format strings for the log output and timestamp
59 formats. It also configures ``lsst.log`` to forward all messages to
60 Python `logging`.
62 This class can perform log uninitialization, which allows command line
63 interface code that initializes logging to run unit tests that execute in
64 batches, without affecting other unit tests. See ``resetLog``."""
66 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
68 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
69 """The log format used when the lsst.log package is not importable and the
70 log is initialized with longlog=True."""
72 pylog_normalFmt = "{name} {levelname}: {message}"
73 """The log format used when the lsst.log package is not importable and the
74 log is initialized with longlog=False."""
76 configState = []
77 """Configuration state. Contains tuples where first item in a tuple is
78 a method and remaining items are arguments for the method.
79 """
81 _initialized = False
82 _componentSettings = []
84 _fileHandlers = []
85 """Any FileHandler classes attached to the root logger by this class
86 that need to be closed on reset."""
88 @classmethod
89 def initLog(cls, longlog: bool, log_tty: bool = True, log_file: Tuple[str, ...] = (),
90 log_label: Optional[Dict[str, str]] = None):
91 """Initialize logging. This should only be called once per program
92 execution. After the first call this will log a warning and return.
94 If lsst.log is importable, will add its log handler to the python
95 root logger's handlers.
97 Parameters
98 ----------
99 longlog : `bool`
100 If True, make log messages appear in long format, by default False.
101 log_tty : `bool`
102 Control whether a default stream handler is enabled that logs
103 to the terminal.
104 log_file : `tuple` of `str`
105 Path to files to write log records. If path ends in ``.json`` the
106 records will be written in JSON format. Else they will be written
107 in text format. If empty no log file will be created. Records
108 will be appended to this file if it exists.
109 log_label : `dict` of `str`
110 Keys and values to be stored in logging MDC for all JSON log
111 records. Keys will be upper-cased.
112 """
113 if cls._initialized:
114 # Unit tests that execute more than one command do end up
115 # calling this function multiple times in one program execution,
116 # so do log a debug but don't log an error or fail, just make the
117 # re-initialization a no-op.
118 log = logging.getLogger(__name__)
119 log.debug("Log is already initialized, returning without re-initializing.")
120 return
121 cls._initialized = True
122 cls._recordComponentSetting(None)
124 if lsstLog is not None:
125 # Ensure that log messages are forwarded back to python.
126 # Disable use of lsst.log MDC -- we expect butler uses to
127 # use ButlerMDC.
128 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
130 # Forward python lsst.log messages directly to python logging.
131 # This can bypass the C++ layer entirely but requires that
132 # MDC is set via ButlerMDC, rather than in lsst.log.
133 lsstLog.usePythonLogging()
135 if not log_tty:
136 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
137 elif longlog:
139 # Want to create our own Formatter so that we can get high
140 # precision timestamps. This requires we attach our own
141 # default stream handler.
142 defaultHandler = logging.StreamHandler()
143 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
144 defaultHandler.setFormatter(formatter)
146 logging.basicConfig(level=logging.INFO,
147 force=True,
148 handlers=[defaultHandler],
149 )
151 else:
152 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{")
154 # Initialize root logger level.
155 cls._setLogLevel(None, "INFO")
157 # also capture warnings and send them to logging
158 logging.captureWarnings(True)
160 # Create a record factory that ensures that an MDC is attached
161 # to the records. By default this is only used for long-log
162 # but always enable it for when someone adds a new handler
163 # that needs it.
164 ButlerMDC.add_mdc_log_record_factory()
166 # Set up the file logger
167 for file in log_file:
168 handler = logging.FileHandler(file)
169 if file.endswith(".json"):
170 formatter = JsonLogFormatter()
171 else:
172 if longlog:
173 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
174 else:
175 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
176 handler.setFormatter(formatter)
177 logging.getLogger().addHandler(handler)
178 cls._fileHandlers.append(handler)
180 # Add any requested MDC records.
181 if log_label:
182 for key, value in log_label.items():
183 ButlerMDC.MDC(key.upper(), value)
185 # remember this call
186 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
188 @classmethod
189 def resetLog(cls):
190 """Uninitialize the butler CLI Log handler and reset component log
191 levels.
193 If the lsst.log handler was added to the python root logger's handlers
194 in `initLog`, it will be removed here.
196 For each logger level that was set by this class, sets that logger's
197 level to the value it was before this class set it. For lsst.log, if a
198 component level was uninitialized, it will be set to
199 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
200 component back to an uninitialized state.
201 """
202 if lsstLog:
203 lsstLog.doNotUsePythonLogging()
204 for componentSetting in reversed(cls._componentSettings):
205 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
206 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
207 logger = logging.getLogger(componentSetting.component)
208 logger.setLevel(componentSetting.pythonLogLevel)
209 cls._setLogLevel(None, "INFO")
211 ButlerMDC.restore_log_record_factory()
213 # Remove the FileHandler we may have attached.
214 root = logging.getLogger()
215 for handler in cls._fileHandlers:
216 handler.close()
217 root.removeHandler(handler)
219 cls._fileHandlers.clear()
220 cls._initialized = False
221 cls.configState = []
223 @classmethod
224 def setLogLevels(cls, logLevels):
225 """Set log level for one or more components or the root logger.
227 Parameters
228 ----------
229 logLevels : `list` of `tuple`
230 per-component logging levels, each item in the list is a tuple
231 (component, level), `component` is a logger name or an empty string
232 or `None` for root logger, `level` is a logging level name, one of
233 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
234 """
235 if isinstance(logLevels, dict):
236 logLevels = logLevels.items()
238 # configure individual loggers
239 for component, level in logLevels:
240 cls._setLogLevel(component, level)
241 # remember this call
242 cls.configState.append((cls._setLogLevel, component, level))
244 @classmethod
245 def _setLogLevel(cls, component, level):
246 """Set the log level for the given component. Record the current log
247 level of the component so that it can be restored when resetting this
248 log.
250 Parameters
251 ----------
252 component : `str` or None
253 The name of the log component or None for the root logger.
254 level : `str`
255 A valid python logging level.
256 """
257 cls._recordComponentSetting(component)
258 if lsstLog is not None:
259 lsstLogger = lsstLog.Log.getLogger(component or "")
260 lsstLogger.setLevel(cls._getLsstLogLevel(level))
261 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
263 @staticmethod
264 def _getPyLogLevel(level):
265 """Get the numeric value for the given log level name.
267 Parameters
268 ----------
269 level : `str`
270 One of the python `logging` log level names.
272 Returns
273 -------
274 numericValue : `int`
275 The python `logging` numeric value for the log level.
276 """
277 if level == "VERBOSE":
278 return VERBOSE
279 return getattr(logging, level, None)
281 @staticmethod
282 def _getLsstLogLevel(level):
283 """Get the numeric value for the given log level name.
285 If `lsst.log` is not setup this function will return `None` regardless
286 of input. `daf_butler` does not directly depend on `lsst.log` and so it
287 will not be setup when `daf_butler` is setup. Packages that depend on
288 `daf_butler` and use `lsst.log` may setup `lsst.log`.
290 Parameters
291 ----------
292 level : `str`
293 One of the python `logging` log level names.
295 Returns
296 -------
297 numericValue : `int` or `None`
298 The `lsst.log` numeric value.
300 Notes
301 -----
302 ``VERBOSE`` logging is not supported by the LSST logger and so will
303 always be converted to ``INFO``.
304 """
305 if lsstLog is None:
306 return None
307 if level == "VERBOSE":
308 level = "INFO"
309 pylog_level = CliLog._getPyLogLevel(level)
310 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level)
312 class ComponentSettings:
313 """Container for log level values for a logging component."""
314 def __init__(self, component):
315 self.component = component
316 self.pythonLogLevel = logging.getLogger(component).level
317 self.lsstLogLevel = (lsstLog.Log.getLogger(component or "").getLevel()
318 if lsstLog is not None else None)
319 if self.lsstLogLevel == -1:
320 self.lsstLogLevel = CliLog.defaultLsstLogLevel
322 def __repr__(self):
323 return (f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
324 f"lsstLogLevel={self.lsstLogLevel})")
326 @classmethod
327 def _recordComponentSetting(cls, component):
328 """Cache current levels for the given component in the list of
329 component levels."""
330 componentSettings = cls.ComponentSettings(component)
331 cls._componentSettings.append(componentSettings)
333 @classmethod
334 def replayConfigState(cls, configState):
335 """Re-create configuration using configuration state recorded earlier.
337 Parameters
338 ----------
339 configState : `list` of `tuple`
340 Tuples contain a method as first item and arguments for the method,
341 in the same format as ``cls.configState``.
342 """
343 if cls._initialized or cls.configState:
344 # Already initialized, do not touch anything.
345 log = logging.getLogger(__name__)
346 log.warning("Log is already initialized, will not replay configuration.")
347 return
349 # execute each one in order
350 for call in configState:
351 method, *args = call
352 method(*args)