Coverage for python/lsst/daf/butler/cli/cliLog.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
24from typing import Tuple, Optional, Dict
26try:
27 import lsst.log as lsstLog
28except ModuleNotFoundError:
29 lsstLog = None
31from ..core.logging import JsonLogFormatter, ButlerMDC
34class PrecisionLogFormatter(logging.Formatter):
35 """A log formatter that issues accurate timezone-aware timestamps."""
37 converter = datetime.datetime.fromtimestamp
39 use_local = True
40 """Control whether local time is displayed instead of UTC."""
42 def formatTime(self, record, datefmt=None):
43 """Format the time as an aware datetime."""
44 ct = self.converter(record.created, tz=datetime.timezone.utc)
45 if self.use_local:
46 ct = ct.astimezone()
47 if datefmt:
48 s = ct.strftime(datefmt)
49 else:
50 s = ct.isoformat(sep='T', timespec='milliseconds')
51 return s
54class CliLog:
55 """Interface for managing python logging and ``lsst.log``.
57 This class defines log format strings for the log output and timestamp
58 formats. It also configures ``lsst.log`` to forward all messages to
59 Python `logging`.
61 This class can perform log uninitialization, which allows command line
62 interface code that initializes logging to run unit tests that execute in
63 batches, without affecting other unit tests. See ``resetLog``."""
65 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
67 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
68 """The log format used when the lsst.log package is not importable and the
69 log is initialized with longlog=True."""
71 pylog_normalFmt = "{name} {levelname}: {message}"
72 """The log format used when the lsst.log package is not importable and the
73 log is initialized with longlog=False."""
75 configState = []
76 """Configuration state. Contains tuples where first item in a tuple is
77 a method and remaining items are arguments for the method.
78 """
80 _initialized = False
81 _componentSettings = []
83 _fileHandlers = []
84 """Any FileHandler classes attached to the root logger by this class
85 that need to be closed on reset."""
87 @classmethod
88 def initLog(cls, longlog: bool, log_tty: bool = True, log_file: Tuple[str, ...] = (),
89 log_label: Optional[Dict[str, str]] = None):
90 """Initialize logging. This should only be called once per program
91 execution. After the first call this will log a warning and return.
93 If lsst.log is importable, will add its log handler to the python
94 root logger's handlers.
96 Parameters
97 ----------
98 longlog : `bool`
99 If True, make log messages appear in long format, by default False.
100 log_tty : `bool`
101 Control whether a default stream handler is enabled that logs
102 to the terminal.
103 log_file : `tuple` of `str`
104 Path to files to write log records. If path ends in ``.json`` the
105 records will be written in JSON format. Else they will be written
106 in text format. If empty no log file will be created. Records
107 will be appended to this file if it exists.
108 log_label : `dict` of `str`
109 Keys and values to be stored in logging MDC for all JSON log
110 records. Keys will be upper-cased.
111 """
112 if cls._initialized:
113 # Unit tests that execute more than one command do end up
114 # calling this function multiple times in one program execution,
115 # so do log a debug but don't log an error or fail, just make the
116 # re-initialization a no-op.
117 log = logging.getLogger(__name__)
118 log.debug("Log is already initialized, returning without re-initializing.")
119 return
120 cls._initialized = True
121 cls._recordComponentSetting(None)
123 if lsstLog is not None:
124 # Ensure that log messages are forwarded back to python.
125 # Disable use of lsst.log MDC -- we expect butler uses to
126 # use ButlerMDC.
127 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
129 # Forward python lsst.log messages directly to python logging.
130 # This can bypass the C++ layer entirely but requires that
131 # MDC is set via ButlerMDC, rather than in lsst.log.
132 lsstLog.usePythonLogging()
134 if not log_tty:
135 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
136 elif longlog:
138 # Want to create our own Formatter so that we can get high
139 # precision timestamps. This requires we attach our own
140 # default stream handler.
141 defaultHandler = logging.StreamHandler()
142 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
143 defaultHandler.setFormatter(formatter)
145 logging.basicConfig(level=logging.INFO,
146 force=True,
147 handlers=[defaultHandler],
148 )
150 else:
151 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{")
153 # Initialize root logger level.
154 cls._setLogLevel(None, "INFO")
156 # also capture warnings and send them to logging
157 logging.captureWarnings(True)
159 # Create a record factory that ensures that an MDC is attached
160 # to the records. By default this is only used for long-log
161 # but always enable it for when someone adds a new handler
162 # that needs it.
163 ButlerMDC.add_mdc_log_record_factory()
165 # Set up the file logger
166 for file in log_file:
167 handler = logging.FileHandler(file)
168 if file.endswith(".json"):
169 formatter = JsonLogFormatter()
170 else:
171 if longlog:
172 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
173 else:
174 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
175 handler.setFormatter(formatter)
176 logging.getLogger().addHandler(handler)
177 cls._fileHandlers.append(handler)
179 # Add any requested MDC records.
180 if log_label:
181 for key, value in log_label.items():
182 ButlerMDC.MDC(key.upper(), value)
184 # remember this call
185 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label))
187 @classmethod
188 def resetLog(cls):
189 """Uninitialize the butler CLI Log handler and reset component log
190 levels.
192 If the lsst.log handler was added to the python root logger's handlers
193 in `initLog`, it will be removed here.
195 For each logger level that was set by this class, sets that logger's
196 level to the value it was before this class set it. For lsst.log, if a
197 component level was uninitialized, it will be set to
198 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
199 component back to an uninitialized state.
200 """
201 if lsstLog:
202 lsstLog.doNotUsePythonLogging()
203 for componentSetting in reversed(cls._componentSettings):
204 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
205 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
206 logger = logging.getLogger(componentSetting.component)
207 logger.setLevel(componentSetting.pythonLogLevel)
208 cls._setLogLevel(None, "INFO")
210 ButlerMDC.restore_log_record_factory()
212 # Remove the FileHandler we may have attached.
213 root = logging.getLogger()
214 for handler in cls._fileHandlers:
215 handler.close()
216 root.removeHandler(handler)
218 cls._fileHandlers.clear()
219 cls._initialized = False
220 cls.configState = []
222 @classmethod
223 def setLogLevels(cls, logLevels):
224 """Set log level for one or more components or the root logger.
226 Parameters
227 ----------
228 logLevels : `list` of `tuple`
229 per-component logging levels, each item in the list is a tuple
230 (component, level), `component` is a logger name or an empty string
231 or `None` for root logger, `level` is a logging level name, one of
232 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
233 """
234 if isinstance(logLevels, dict):
235 logLevels = logLevels.items()
237 # configure individual loggers
238 for component, level in logLevels:
239 cls._setLogLevel(component, level)
240 # remember this call
241 cls.configState.append((cls._setLogLevel, component, level))
243 @classmethod
244 def _setLogLevel(cls, component, level):
245 """Set the log level for the given component. Record the current log
246 level of the component so that it can be restored when resetting this
247 log.
249 Parameters
250 ----------
251 component : `str` or None
252 The name of the log component or None for the root logger.
253 level : `str`
254 A valid python logging level.
255 """
256 cls._recordComponentSetting(component)
257 if lsstLog is not None:
258 lsstLogger = lsstLog.Log.getLogger(component or "")
259 lsstLogger.setLevel(cls._getLsstLogLevel(level))
260 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
262 @staticmethod
263 def _getPyLogLevel(level):
264 """Get the numeric value for the given log level name.
266 Parameters
267 ----------
268 level : `str`
269 One of the python `logging` log level names.
271 Returns
272 -------
273 numericValue : `int`
274 The python `logging` numeric value for the log level.
275 """
276 if level == "VERBOSE":
277 from .. import VERBOSE
278 return VERBOSE
279 return getattr(logging, level, None)
281 @staticmethod
282 def _getLsstLogLevel(level):
283 """Get the numeric value for the given log level name.
285 If `lsst.log` is not setup this function will return `None` regardless
286 of input. `daf_butler` does not directly depend on `lsst.log` and so it
287 will not be setup when `daf_butler` is setup. Packages that depend on
288 `daf_butler` and use `lsst.log` may setup `lsst.log`.
290 Will adapt the python name to an `lsst.log` name:
291 - CRITICAL to FATAL
292 - WARNING to WARN
294 Parameters
295 ----------
296 level : `str`
297 One of the python `logging` log level names.
299 Returns
300 -------
301 numericValue : `int` or `None`
302 The `lsst.log` numeric value.
303 """
304 if lsstLog is None:
305 return None
306 if level == "CRITICAL":
307 level = "FATAL"
308 elif level == "WARNING":
309 level = "WARN"
310 elif level == "VERBOSE":
311 # LSST log does not yet have verbose defined
312 return (lsstLog.Log.DEBUG + lsstLog.Log.INFO) // 2
313 return getattr(lsstLog.Log, level, None)
315 class ComponentSettings:
316 """Container for log level values for a logging component."""
317 def __init__(self, component):
318 self.component = component
319 self.pythonLogLevel = logging.getLogger(component).level
320 self.lsstLogLevel = (lsstLog.Log.getLogger(component or "").getLevel()
321 if lsstLog is not None else None)
322 if self.lsstLogLevel == -1:
323 self.lsstLogLevel = CliLog.defaultLsstLogLevel
325 def __repr__(self):
326 return (f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
327 f"lsstLogLevel={self.lsstLogLevel})")
329 @classmethod
330 def _recordComponentSetting(cls, component):
331 """Cache current levels for the given component in the list of
332 component levels."""
333 componentSettings = cls.ComponentSettings(component)
334 cls._componentSettings.append(componentSettings)
336 @classmethod
337 def replayConfigState(cls, configState):
338 """Re-create configuration using configuration state recorded earlier.
340 Parameters
341 ----------
342 configState : `list` of `tuple`
343 Tuples contain a method as first item and arguments for the method,
344 in the same format as ``cls.configState``.
345 """
346 if cls._initialized or cls.configState:
347 # Already initialized, do not touch anything.
348 log = logging.getLogger(__name__)
349 log.warning("Log is already initialized, will not replay configuration.")
350 return
352 # execute each one in order
353 for call in configState:
354 method, *args = call
355 method(*args)