Coverage for python/lsst/daf/butler/cli/cliLog.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import datetime
23import logging
25try:
26 import lsst.log as lsstLog
27except ModuleNotFoundError:
28 lsstLog = None
30from lsst.daf.butler import ButlerMDC
31from ..core.logging import MDCDict
34class PrecisionLogFormatter(logging.Formatter):
35 """A log formatter that issues accurate timezone-aware timestamps."""
37 converter = datetime.datetime.fromtimestamp
39 use_local = True
40 """Control whether local time is displayed instead of UTC."""
42 def formatTime(self, record, datefmt=None):
43 """Format the time as an aware datetime."""
44 ct = self.converter(record.created, tz=datetime.timezone.utc)
45 if self.use_local:
46 ct = ct.astimezone()
47 if datefmt:
48 s = ct.strftime(datefmt)
49 else:
50 s = ct.isoformat(sep='T', timespec='milliseconds')
51 return s
54class CliLog:
55 """Interface for managing python logging and ``lsst.log``.
57 This class defines log format strings for the log output and timestamp
58 formats. It also configures ``lsst.log`` to forward all messages to
59 Python `logging`.
61 This class can perform log uninitialization, which allows command line
62 interface code that initializes logging to run unit tests that execute in
63 batches, without affecting other unit tests. See ``resetLog``."""
65 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None
67 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}"
68 """The log format used when the lsst.log package is not importable and the
69 log is initialized with longlog=True."""
71 pylog_normalFmt = "{name} {levelname}: {message}"
72 """The log format used when the lsst.log package is not importable and the
73 log is initialized with longlog=False."""
75 configState = []
76 """Configuration state. Contains tuples where first item in a tuple is
77 a method and remaining items are arguments for the method.
78 """
80 _initialized = False
81 _componentSettings = []
83 @classmethod
84 def initLog(cls, longlog):
85 """Initialize logging. This should only be called once per program
86 execution. After the first call this will log a warning and return.
88 If lsst.log is importable, will add its log handler to the python
89 root logger's handlers.
91 Parameters
92 ----------
93 longlog : `bool`
94 If True, make log messages appear in long format, by default False.
95 """
96 if cls._initialized:
97 # Unit tests that execute more than one command do end up
98 # calling this function multiple times in one program execution,
99 # so do log a debug but don't log an error or fail, just make the
100 # re-initialization a no-op.
101 log = logging.getLogger(__name__)
102 log.debug("Log is already initialized, returning without re-initializing.")
103 return
104 cls._initialized = True
105 cls._recordComponentSetting(None)
107 if lsstLog is not None:
108 # Ensure that log messages are forwarded back to python.
109 # Disable use of lsst.log MDC -- we expect butler uses to
110 # use ButlerMDC.
111 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None)
113 # Forward python lsst.log messages directly to python logging.
114 # This can bypass the C++ layer entirely but requires that
115 # MDC is set via ButlerMDC, rather than in lsst.log.
116 lsstLog.usePythonLogging()
118 if longlog:
120 # Want to create our own Formatter so that we can get high
121 # precision timestamps. This requires we attach our own
122 # default stream handler.
123 defaultHandler = logging.StreamHandler()
124 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{")
125 defaultHandler.setFormatter(formatter)
127 logging.basicConfig(level=logging.INFO,
128 force=True,
129 handlers=[defaultHandler],
130 )
132 else:
133 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{")
135 # Initialize root logger level.
136 cls._setLogLevel(None, "INFO")
138 # also capture warnings and send them to logging
139 logging.captureWarnings(True)
141 # Create a record factory that ensures that an MDC is attached
142 # to the records. By default this is only used for long-log
143 # but always enable it for when someone adds a new handler
144 # that needs it.
145 old_factory = logging.getLogRecordFactory()
147 def record_factory(*args, **kwargs):
148 record = old_factory(*args, **kwargs)
149 # Make sure we send a copy of the global dict in the record.
150 record.MDC = MDCDict(ButlerMDC._MDC)
151 return record
153 logging.setLogRecordFactory(record_factory)
155 # remember this call
156 cls.configState.append((cls.initLog, longlog))
158 @classmethod
159 def resetLog(cls):
160 """Uninitialize the butler CLI Log handler and reset component log
161 levels.
163 If the lsst.log handler was added to the python root logger's handlers
164 in `initLog`, it will be removed here.
166 For each logger level that was set by this class, sets that logger's
167 level to the value it was before this class set it. For lsst.log, if a
168 component level was uninitialized, it will be set to
169 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a
170 component back to an uninitialized state.
171 """
172 if lsstLog:
173 lsstLog.doNotUsePythonLogging()
174 for componentSetting in reversed(cls._componentSettings):
175 if lsstLog is not None and componentSetting.lsstLogLevel is not None:
176 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel)
177 logger = logging.getLogger(componentSetting.component)
178 logger.setLevel(componentSetting.pythonLogLevel)
179 cls._setLogLevel(None, "INFO")
180 cls._initialized = False
181 cls.configState = []
183 @classmethod
184 def setLogLevels(cls, logLevels):
185 """Set log level for one or more components or the root logger.
187 Parameters
188 ----------
189 logLevels : `list` of `tuple`
190 per-component logging levels, each item in the list is a tuple
191 (component, level), `component` is a logger name or an empty string
192 or `None` for root logger, `level` is a logging level name, one of
193 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
194 """
195 if isinstance(logLevels, dict):
196 logLevels = logLevels.items()
198 # configure individual loggers
199 for component, level in logLevels:
200 cls._setLogLevel(component, level)
201 # remember this call
202 cls.configState.append((cls._setLogLevel, component, level))
204 @classmethod
205 def _setLogLevel(cls, component, level):
206 """Set the log level for the given component. Record the current log
207 level of the component so that it can be restored when resetting this
208 log.
210 Parameters
211 ----------
212 component : `str` or None
213 The name of the log component or None for the root logger.
214 level : `str`
215 A valid python logging level.
216 """
217 cls._recordComponentSetting(component)
218 if lsstLog is not None:
219 lsstLogger = lsstLog.Log.getLogger(component or "")
220 lsstLogger.setLevel(cls._getLsstLogLevel(level))
221 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level))
223 @staticmethod
224 def _getPyLogLevel(level):
225 """Get the numeric value for the given log level name.
227 Parameters
228 ----------
229 level : `str`
230 One of the python `logging` log level names.
232 Returns
233 -------
234 numericValue : `int`
235 The python `logging` numeric value for the log level.
236 """
237 if level == "VERBOSE":
238 from .. import VERBOSE
239 return VERBOSE
240 return getattr(logging, level, None)
242 @staticmethod
243 def _getLsstLogLevel(level):
244 """Get the numeric value for the given log level name.
246 If `lsst.log` is not setup this function will return `None` regardless
247 of input. `daf_butler` does not directly depend on `lsst.log` and so it
248 will not be setup when `daf_butler` is setup. Packages that depend on
249 `daf_butler` and use `lsst.log` may setup `lsst.log`.
251 Will adapt the python name to an `lsst.log` name:
252 - CRITICAL to FATAL
253 - WARNING to WARN
255 Parameters
256 ----------
257 level : `str`
258 One of the python `logging` log level names.
260 Returns
261 -------
262 numericValue : `int` or `None`
263 The `lsst.log` numeric value.
264 """
265 if lsstLog is None:
266 return None
267 if level == "CRITICAL":
268 level = "FATAL"
269 elif level == "WARNING":
270 level = "WARN"
271 elif level == "VERBOSE":
272 # LSST log does not yet have verbose defined
273 return (lsstLog.Log.DEBUG + lsstLog.Log.INFO) // 2
274 return getattr(lsstLog.Log, level, None)
276 class ComponentSettings:
277 """Container for log level values for a logging component."""
278 def __init__(self, component):
279 self.component = component
280 self.pythonLogLevel = logging.getLogger(component).level
281 self.lsstLogLevel = (lsstLog.Log.getLogger(component or "").getLevel()
282 if lsstLog is not None else None)
283 if self.lsstLogLevel == -1:
284 self.lsstLogLevel = CliLog.defaultLsstLogLevel
286 def __repr__(self):
287 return (f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, "
288 f"lsstLogLevel={self.lsstLogLevel})")
290 @classmethod
291 def _recordComponentSetting(cls, component):
292 """Cache current levels for the given component in the list of
293 component levels."""
294 componentSettings = cls.ComponentSettings(component)
295 cls._componentSettings.append(componentSettings)
297 @classmethod
298 def replayConfigState(cls, configState):
299 """Re-create configuration using configuration state recorded earlier.
301 Parameters
302 ----------
303 configState : `list` of `tuple`
304 Tuples contain a method as first item and arguments for the method,
305 in the same format as ``cls.configState``.
306 """
307 if cls._initialized or cls.configState:
308 # Already initialized, do not touch anything.
309 log = logging.getLogger(__name__)
310 log.warning("Log is already initialized, will not replay configuration.")
311 return
313 # execute each one in order
314 for call in configState:
315 method, *args = call
316 method(*args)