Coverage for python/lsst/daf/butler/cli/cliLog.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
25try:
26 import lsst.log as lsstLog
27except ModuleNotFoundError:
28 lsstLog = None
30from ..core.logging import JsonLogFormatter, ButlerMDC
33class PrecisionLogFormatter(logging.Formatter):
34 """A log formatter that issues accurate timezone-aware timestamps."""
36 converter = datetime.datetime.fromtimestamp
38 use_local = True
39 """Control whether local time is displayed instead of UTC."""
41 def formatTime(self, record, datefmt=None):
42 """Format the time as an aware datetime."""
43 ct = self.converter(record.created, tz=datetime.timezone.utc)
44 if self.use_local:
45 ct = ct.astimezone()
46 if datefmt:
47 s = ct.strftime(datefmt)
48 else:
49 s = ct.isoformat(sep='T', timespec='milliseconds')
50 return s
53class CliLog:
54 """Interface for managing python logging and ``lsst.log``.
56 This class defines log format strings for the log output and timestamp
57 formats. It also configures ``lsst.log`` to forward all messages to
58 Python `logging`.
60 This class can perform log uninitialization, which allows command line
61 interface code that initializes logging to run unit tests that execute in
62 batches, without affecting other unit tests. See ``resetLog``."""
64 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
66 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
67 """The log format used when the lsst.log package is not importable and the
68 log is initialized with longlog=True."""
70 pylog_normalFmt = "{name} {levelname}: {message}"
71 """The log format used when the lsst.log package is not importable and the
72 log is initialized with longlog=False."""
74 configState = []
75 """Configuration state. Contains tuples where first item in a tuple is
76 a method and remaining items are arguments for the method.
77 """
79 _initialized = False
80 _componentSettings = []
82 _fileHandlers = []
83 """Any FileHandler classes attached to the root logger by this class
84 that need to be closed on reset."""
86 @classmethod
87 def initLog(cls, longlog: bool, log_tty: bool = True, log_file=()):
88 """Initialize logging. This should only be called once per program
89 execution. After the first call this will log a warning and return.
91 If lsst.log is importable, will add its log handler to the python
92 root logger's handlers.
94 Parameters
95 ----------
96 longlog : `bool`
97 If True, make log messages appear in long format, by default False.
98 log_tty : `bool`
99 Control whether a default stream handler is enabled that logs
100 to the terminal.
101 log_file : `tuple` of `str`
102 Path to files to write log records. If path ends in ``.json`` the
103 records will be written in JSON format. Else they will be written
104 in text format. If empty no log file will be created. Records
105 will be appended to this file if it exists.
106 """
107 if cls._initialized:
108 # Unit tests that execute more than one command do end up
109 # calling this function multiple times in one program execution,
110 # so do log a debug but don't log an error or fail, just make the
111 # re-initialization a no-op.
112 log = logging.getLogger(__name__)
113 log.debug("Log is already initialized, returning without re-initializing.")
114 return
115 cls._initialized = True
116 cls._recordComponentSetting(None)
118 if lsstLog is not None:
119 # Ensure that log messages are forwarded back to python.
120 # Disable use of lsst.log MDC -- we expect butler uses to
121 # use ButlerMDC.
122 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
124 # Forward python lsst.log messages directly to python logging.
125 # This can bypass the C++ layer entirely but requires that
126 # MDC is set via ButlerMDC, rather than in lsst.log.
127 lsstLog.usePythonLogging()
129 if not log_tty:
130 logging.basicConfig(force=True, handlers=[logging.NullHandler()])
131 elif longlog:
133 # Want to create our own Formatter so that we can get high
134 # precision timestamps. This requires we attach our own
135 # default stream handler.
136 defaultHandler = logging.StreamHandler()
137 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
138 defaultHandler.setFormatter(formatter)
140 logging.basicConfig(level=logging.INFO,
141 force=True,
142 handlers=[defaultHandler],
143 )
145 else:
146 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{")
148 # Initialize root logger level.
149 cls._setLogLevel(None, "INFO")
151 # also capture warnings and send them to logging
152 logging.captureWarnings(True)
154 # Create a record factory that ensures that an MDC is attached
155 # to the records. By default this is only used for long-log
156 # but always enable it for when someone adds a new handler
157 # that needs it.
158 ButlerMDC.add_mdc_log_record_factory()
160 # Set up the file logger
161 for file in log_file:
162 handler = logging.FileHandler(file)
163 if file.endswith(".json"):
164 formatter = JsonLogFormatter()
165 else:
166 if longlog:
167 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
168 else:
169 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{")
170 handler.setFormatter(formatter)
171 logging.getLogger().addHandler(handler)
172 cls._fileHandlers.append(handler)
174 # remember this call
175 cls.configState.append((cls.initLog, longlog, log_tty, log_file))
177 @classmethod
178 def resetLog(cls):
179 """Uninitialize the butler CLI Log handler and reset component log
180 levels.
182 If the lsst.log handler was added to the python root logger's handlers
183 in `initLog`, it will be removed here.
185 For each logger level that was set by this class, sets that logger's
186 level to the value it was before this class set it. For lsst.log, if a
187 component level was uninitialized, it will be set to
188 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
189 component back to an uninitialized state.
190 """
191 if lsstLog:
192 lsstLog.doNotUsePythonLogging()
193 for componentSetting in reversed(cls._componentSettings):
194 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
195 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
196 logger = logging.getLogger(componentSetting.component)
197 logger.setLevel(componentSetting.pythonLogLevel)
198 cls._setLogLevel(None, "INFO")
200 ButlerMDC.restore_log_record_factory()
202 # Remove the FileHandler we may have attached.
203 root = logging.getLogger()
204 for handler in cls._fileHandlers:
205 handler.close()
206 root.removeHandler(handler)
208 cls._fileHandlers.clear()
209 cls._initialized = False
210 cls.configState = []
212 @classmethod
213 def setLogLevels(cls, logLevels):
214 """Set log level for one or more components or the root logger.
216 Parameters
217 ----------
218 logLevels : `list` of `tuple`
219 per-component logging levels, each item in the list is a tuple
220 (component, level), `component` is a logger name or an empty string
221 or `None` for root logger, `level` is a logging level name, one of
222 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
223 """
224 if isinstance(logLevels, dict):
225 logLevels = logLevels.items()
227 # configure individual loggers
228 for component, level in logLevels:
229 cls._setLogLevel(component, level)
230 # remember this call
231 cls.configState.append((cls._setLogLevel, component, level))
233 @classmethod
234 def _setLogLevel(cls, component, level):
235 """Set the log level for the given component. Record the current log
236 level of the component so that it can be restored when resetting this
237 log.
239 Parameters
240 ----------
241 component : `str` or None
242 The name of the log component or None for the root logger.
243 level : `str`
244 A valid python logging level.
245 """
246 cls._recordComponentSetting(component)
247 if lsstLog is not None:
248 lsstLogger = lsstLog.Log.getLogger(component or "")
249 lsstLogger.setLevel(cls._getLsstLogLevel(level))
250 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
252 @staticmethod
253 def _getPyLogLevel(level):
254 """Get the numeric value for the given log level name.
256 Parameters
257 ----------
258 level : `str`
259 One of the python `logging` log level names.
261 Returns
262 -------
263 numericValue : `int`
264 The python `logging` numeric value for the log level.
265 """
266 if level == "VERBOSE":
267 from .. import VERBOSE
268 return VERBOSE
269 return getattr(logging, level, None)
271 @staticmethod
272 def _getLsstLogLevel(level):
273 """Get the numeric value for the given log level name.
275 If `lsst.log` is not setup this function will return `None` regardless
276 of input. `daf_butler` does not directly depend on `lsst.log` and so it
277 will not be setup when `daf_butler` is setup. Packages that depend on
278 `daf_butler` and use `lsst.log` may setup `lsst.log`.
280 Will adapt the python name to an `lsst.log` name:
281 - CRITICAL to FATAL
282 - WARNING to WARN
284 Parameters
285 ----------
286 level : `str`
287 One of the python `logging` log level names.
289 Returns
290 -------
291 numericValue : `int` or `None`
292 The `lsst.log` numeric value.
293 """
294 if lsstLog is None:
295 return None
296 if level == "CRITICAL":
297 level = "FATAL"
298 elif level == "WARNING":
299 level = "WARN"
300 elif level == "VERBOSE":
301 # LSST log does not yet have verbose defined
302 return (lsstLog.Log.DEBUG + lsstLog.Log.INFO) // 2
303 return getattr(lsstLog.Log, level, None)
305 class ComponentSettings:
306 """Container for log level values for a logging component."""
307 def __init__(self, component):
308 self.component = component
309 self.pythonLogLevel = logging.getLogger(component).level
310 self.lsstLogLevel = (lsstLog.Log.getLogger(component or "").getLevel()
311 if lsstLog is not None else None)
312 if self.lsstLogLevel == -1:
313 self.lsstLogLevel = CliLog.defaultLsstLogLevel
315 def __repr__(self):
316 return (f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
317 f"lsstLogLevel={self.lsstLogLevel})")
319 @classmethod
320 def _recordComponentSetting(cls, component):
321 """Cache current levels for the given component in the list of
322 component levels."""
323 componentSettings = cls.ComponentSettings(component)
324 cls._componentSettings.append(componentSettings)
326 @classmethod
327 def replayConfigState(cls, configState):
328 """Re-create configuration using configuration state recorded earlier.
330 Parameters
331 ----------
332 configState : `list` of `tuple`
333 Tuples contain a method as first item and arguments for the method,
334 in the same format as ``cls.configState``.
335 """
336 if cls._initialized or cls.configState:
337 # Already initialized, do not touch anything.
338 log = logging.getLogger(__name__)
339 log.warning("Log is already initialized, will not replay configuration.")
340 return
342 # execute each one in order
343 for call in configState:
344 method, *args = call
345 method(*args)