Coverage for python/lsst/daf/butler/cli/cliLog.py: 30%

159 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "PrecisionLogFormatter", 

31 "CliLog", 

32) 

33 

34import datetime 

35import logging 

36import os 

37from typing import Any 

38 

39try: 

40 import lsst.log as lsstLog 

41except ModuleNotFoundError: 

42 lsstLog = None 

43 

44from lsst.utils.logging import TRACE, VERBOSE 

45 

46from ..core.logging import ButlerMDC, JsonLogFormatter 

47 

48 

49class PrecisionLogFormatter(logging.Formatter): 

50 """A log formatter that issues accurate timezone-aware timestamps.""" 

51 

52 converter = datetime.datetime.fromtimestamp # type: ignore 

53 

54 use_local = True 

55 """Control whether local time is displayed instead of UTC.""" 

56 

57 def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: 

58 """Format the time as an aware datetime.""" 

59 ct: datetime.datetime = self.converter(record.created, tz=datetime.timezone.utc) # type: ignore 

60 if self.use_local: 

61 ct = ct.astimezone() 

62 if datefmt: 

63 s = ct.strftime(datefmt) 

64 else: 

65 s = ct.isoformat(sep="T", timespec="milliseconds") 

66 return s 

67 

68 

69class CliLog: 

70 """Interface for managing python logging and ``lsst.log``. 

71 

72 This class defines log format strings for the log output and timestamp 

73 formats. It also configures ``lsst.log`` to forward all messages to 

74 Python `logging`. 

75 

76 This class can perform log uninitialization, which allows command line 

77 interface code that initializes logging to run unit tests that execute in 

78 batches, without affecting other unit tests. See ``resetLog``. 

79 """ 

80 

81 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

82 

83 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

84 """The log format used when the lsst.log package is not importable and the 

85 log is initialized with longlog=True.""" 

86 

87 pylog_normalFmt = "{name} {levelname}: {message}" 

88 """The log format used when the lsst.log package is not importable and the 

89 log is initialized with longlog=False.""" 

90 

91 configState: list[tuple[Any, ...]] = [] 

92 """Configuration state. Contains tuples where first item in a tuple is 

93 a method and remaining items are arguments for the method. 

94 """ 

95 

96 _initialized = False 

97 _componentSettings: list[ComponentSettings] = [] 

98 

99 _fileHandlers: list[logging.FileHandler] = [] 

100 """Any FileHandler classes attached to the root logger by this class 

101 that need to be closed on reset.""" 

102 

103 @staticmethod 

104 def root_loggers() -> set[str]: 

105 """Return the default root logger. 

106 

107 Returns 

108 ------- 

109 log_name : `set` of `str` 

110 The name(s) of the root logger(s) to use when the log level is 

111 being set without a log name being specified. 

112 

113 Notes 

114 ----- 

115 The default is ``lsst`` (which controls the butler infrastructure) 

116 but additional loggers can be specified by setting the environment 

117 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain 

118 multiple default loggers separated by a ``:``. 

119 """ 

120 log_names = {"lsst"} 

121 envvar = "DAF_BUTLER_ROOT_LOGGER" 

122 if envvar in os.environ: 122 ↛ 123line 122 didn't jump to line 123, because the condition on line 122 was never true

123 log_names |= set(os.environ[envvar].split(":")) 

124 return log_names 

125 

126 @classmethod 

127 def initLog( 

128 cls, 

129 longlog: bool, 

130 log_tty: bool = True, 

131 log_file: tuple[str, ...] = (), 

132 log_label: dict[str, str] | None = None, 

133 ) -> None: 

134 """Initialize logging. This should only be called once per program 

135 execution. After the first call this will log a warning and return. 

136 

137 If lsst.log is importable, will add its log handler to the python 

138 root logger's handlers. 

139 

140 Parameters 

141 ---------- 

142 longlog : `bool` 

143 If True, make log messages appear in long format, by default False. 

144 log_tty : `bool` 

145 Control whether a default stream handler is enabled that logs 

146 to the terminal. 

147 log_file : `tuple` of `str` 

148 Path to files to write log records. If path ends in ``.json`` the 

149 records will be written in JSON format. Else they will be written 

150 in text format. If empty no log file will be created. Records 

151 will be appended to this file if it exists. 

152 log_label : `dict` of `str` 

153 Keys and values to be stored in logging MDC for all JSON log 

154 records. Keys will be upper-cased. 

155 """ 

156 if cls._initialized: 

157 # Unit tests that execute more than one command do end up 

158 # calling this function multiple times in one program execution, 

159 # so do log a debug but don't log an error or fail, just make the 

160 # re-initialization a no-op. 

161 log = logging.getLogger(__name__) 

162 log.debug("Log is already initialized, returning without re-initializing.") 

163 return 

164 cls._initialized = True 

165 cls._recordComponentSetting(None) 

166 

167 if lsstLog is not None: 

168 # Ensure that log messages are forwarded back to python. 

169 # Disable use of lsst.log MDC -- we expect butler uses to 

170 # use ButlerMDC. 

171 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

172 

173 # Forward python lsst.log messages directly to python logging. 

174 # This can bypass the C++ layer entirely but requires that 

175 # MDC is set via ButlerMDC, rather than in lsst.log. 

176 lsstLog.usePythonLogging() 

177 

178 formatter: logging.Formatter 

179 if not log_tty: 

180 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

181 elif longlog: 

182 # Want to create our own Formatter so that we can get high 

183 # precision timestamps. This requires we attach our own 

184 # default stream handler. 

185 defaultHandler = logging.StreamHandler() 

186 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

187 defaultHandler.setFormatter(formatter) 

188 

189 logging.basicConfig( 

190 level=logging.WARNING, 

191 force=True, 

192 handlers=[defaultHandler], 

193 ) 

194 

195 else: 

196 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{") 

197 

198 # Initialize the root logger. Calling this ensures that both 

199 # python loggers and lsst loggers are consistent in their default 

200 # logging level. 

201 cls._setLogLevel(".", "WARNING") 

202 

203 # Initialize default root logger level. 

204 cls._setLogLevel(None, "INFO") 

205 

206 # also capture warnings and send them to logging 

207 logging.captureWarnings(True) 

208 

209 # Create a record factory that ensures that an MDC is attached 

210 # to the records. By default this is only used for long-log 

211 # but always enable it for when someone adds a new handler 

212 # that needs it. 

213 ButlerMDC.add_mdc_log_record_factory() 

214 

215 # Set up the file logger 

216 for file in log_file: 

217 handler = logging.FileHandler(file) 

218 if file.endswith(".json"): 

219 formatter = JsonLogFormatter() 

220 else: 

221 if longlog: 

222 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

223 else: 

224 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

225 handler.setFormatter(formatter) 

226 logging.getLogger().addHandler(handler) 

227 cls._fileHandlers.append(handler) 

228 

229 # Add any requested MDC records. 

230 if log_label: 

231 for key, value in log_label.items(): 

232 ButlerMDC.MDC(key.upper(), value) 

233 

234 # remember this call 

235 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

236 

237 @classmethod 

238 def resetLog(cls) -> None: 

239 """Uninitialize the butler CLI Log handler and reset component log 

240 levels. 

241 

242 If the lsst.log handler was added to the python root logger's handlers 

243 in `initLog`, it will be removed here. 

244 

245 For each logger level that was set by this class, sets that logger's 

246 level to the value it was before this class set it. For lsst.log, if a 

247 component level was uninitialized, it will be set to 

248 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

249 component back to an uninitialized state. 

250 """ 

251 if lsstLog: 

252 lsstLog.doNotUsePythonLogging() 

253 for componentSetting in reversed(cls._componentSettings): 

254 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

255 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

256 logger = logging.getLogger(componentSetting.component) 

257 logger.setLevel(componentSetting.pythonLogLevel) 

258 cls._setLogLevel(None, "INFO") 

259 

260 ButlerMDC.restore_log_record_factory() 

261 

262 # Remove the FileHandler we may have attached. 

263 root = logging.getLogger() 

264 for handler in cls._fileHandlers: 

265 handler.close() 

266 root.removeHandler(handler) 

267 

268 cls._fileHandlers.clear() 

269 cls._initialized = False 

270 cls.configState = [] 

271 

272 @classmethod 

273 def setLogLevels(cls, logLevels: list[tuple[str | None, str]] | dict[str, str]) -> None: 

274 """Set log level for one or more components or the root logger. 

275 

276 Parameters 

277 ---------- 

278 logLevels : `list` of `tuple` 

279 per-component logging levels, each item in the list is a tuple 

280 (component, level), `component` is a logger name or an empty string 

281 or `None` for default root logger, `level` is a logging level name, 

282 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

283 

284 Notes 

285 ----- 

286 The special name ``.`` can be used to set the Python root 

287 logger. 

288 """ 

289 if isinstance(logLevels, dict): 

290 logLevels = list(logLevels.items()) 

291 

292 # configure individual loggers 

293 for component, level in logLevels: 

294 cls._setLogLevel(component, level) 

295 # remember this call 

296 cls.configState.append((cls._setLogLevel, component, level)) 

297 

298 @classmethod 

299 def _setLogLevel(cls, component: str | None, level: str) -> None: 

300 """Set the log level for the given component. Record the current log 

301 level of the component so that it can be restored when resetting this 

302 log. 

303 

304 Parameters 

305 ---------- 

306 component : `str` or None 

307 The name of the log component or None for the default logger. 

308 The root logger can be specified either by an empty string or 

309 with the special name ``.``. 

310 level : `str` 

311 A valid python logging level. 

312 """ 

313 components: set[str | None] 

314 if component is None: 

315 components = set(cls.root_loggers()) 

316 elif not component or component == ".": 

317 components = {None} 

318 else: 

319 components = {component} 

320 for component in components: 

321 cls._recordComponentSetting(component) 

322 if lsstLog is not None: 

323 lsstLogger = lsstLog.Log.getLogger(component or "") 

324 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

325 pylevel = cls._getPyLogLevel(level) 

326 if pylevel is not None: 

327 logging.getLogger(component or None).setLevel(pylevel) 

328 

329 @staticmethod 

330 def _getPyLogLevel(level: str) -> int | None: 

331 """Get the numeric value for the given log level name. 

332 

333 Parameters 

334 ---------- 

335 level : `str` 

336 One of the python `logging` log level names. 

337 

338 Returns 

339 ------- 

340 numericValue : `int` 

341 The python `logging` numeric value for the log level. 

342 """ 

343 if level == "VERBOSE": 

344 return VERBOSE 

345 elif level == "TRACE": 

346 return TRACE 

347 return getattr(logging, level, None) 

348 

349 @staticmethod 

350 def _getLsstLogLevel(level: str) -> int | None: 

351 """Get the numeric value for the given log level name. 

352 

353 If `lsst.log` is not setup this function will return `None` regardless 

354 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

355 will not be setup when `daf_butler` is setup. Packages that depend on 

356 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

357 

358 Parameters 

359 ---------- 

360 level : `str` 

361 One of the python `logging` log level names. 

362 

363 Returns 

364 ------- 

365 numericValue : `int` or `None` 

366 The `lsst.log` numeric value. 

367 

368 Notes 

369 ----- 

370 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger. 

371 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be 

372 converted to ``DEBUG``. 

373 """ 

374 if lsstLog is None: 

375 return None 

376 if level == "VERBOSE": 

377 level = "INFO" 

378 elif level == "TRACE": 

379 level = "DEBUG" 

380 pylog_level = CliLog._getPyLogLevel(level) 

381 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

382 

383 class ComponentSettings: 

384 """Container for log level values for a logging component.""" 

385 

386 def __init__(self, component: str | None): 

387 self.component = component 

388 self.pythonLogLevel = logging.getLogger(component).level 

389 self.lsstLogLevel = ( 

390 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

391 ) 

392 if self.lsstLogLevel == -1: 

393 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

394 

395 def __repr__(self) -> str: 

396 return ( 

397 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

398 f"lsstLogLevel={self.lsstLogLevel})" 

399 ) 

400 

401 @classmethod 

402 def _recordComponentSetting(cls, component: str | None) -> None: 

403 """Cache current levels for the given component in the list of 

404 component levels. 

405 """ 

406 componentSettings = cls.ComponentSettings(component) 

407 cls._componentSettings.append(componentSettings) 

408 

409 @classmethod 

410 def replayConfigState(cls, configState: list[tuple[Any, ...]]) -> None: 

411 """Re-create configuration using configuration state recorded earlier. 

412 

413 Parameters 

414 ---------- 

415 configState : `list` of `tuple` 

416 Tuples contain a method as first item and arguments for the method, 

417 in the same format as ``cls.configState``. 

418 """ 

419 if cls._initialized or cls.configState: 

420 # Already initialized, do not touch anything. 

421 log = logging.getLogger(__name__) 

422 log.warning("Log is already initialized, will not replay configuration.") 

423 return 

424 

425 # execute each one in order 

426 for call in configState: 

427 method, *args = call 

428 method(*args)