Coverage for python/lsst/daf/butler/cli/cliLog.py: 30%

159 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-02 03:16 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "PrecisionLogFormatter", 

31 "CliLog", 

32) 

33 

34import datetime 

35import logging 

36import os 

37from typing import Any 

38 

39try: 

40 import lsst.log as lsstLog 

41except ModuleNotFoundError: 

42 lsstLog = None 

43 

44from lsst.utils.logging import TRACE, VERBOSE 

45 

46from ..logging import ButlerMDC, JsonLogFormatter 

47 

48 

49class PrecisionLogFormatter(logging.Formatter): 

50 """A log formatter that issues accurate timezone-aware timestamps.""" 

51 

52 converter = datetime.datetime.fromtimestamp # type: ignore 

53 

54 use_local = True 

55 """Control whether local time is displayed instead of UTC.""" 

56 

57 def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: 

58 """Format the time as an aware datetime. 

59 

60 Parameters 

61 ---------- 

62 record : `logging.LogRecord` 

63 The record to format. 

64 datefmt : `str` or `None`, optional 

65 Format to use when formatting the date. 

66 

67 Returns 

68 ------- 

69 formatted : `str` 

70 Formatted date string. 

71 """ 

72 ct: datetime.datetime = self.converter(record.created, tz=datetime.UTC) # type: ignore 

73 if self.use_local: 

74 ct = ct.astimezone() 

75 if datefmt: 

76 s = ct.strftime(datefmt) 

77 else: 

78 s = ct.isoformat(sep="T", timespec="milliseconds") 

79 return s 

80 

81 

82class CliLog: 

83 """Interface for managing python logging and ``lsst.log``. 

84 

85 This class defines log format strings for the log output and timestamp 

86 formats. It also configures ``lsst.log`` to forward all messages to 

87 Python `logging`. 

88 

89 This class can perform log uninitialization, which allows command line 

90 interface code that initializes logging to run unit tests that execute in 

91 batches, without affecting other unit tests. See ``resetLog``. 

92 """ 

93 

94 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

95 

96 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

97 """The log format used when the lsst.log package is not importable and the 

98 log is initialized with longlog=True.""" 

99 

100 pylog_normalFmt = "{name} {levelname}: {message}" 

101 """The log format used when the lsst.log package is not importable and the 

102 log is initialized with longlog=False.""" 

103 

104 configState: list[tuple[Any, ...]] = [] 

105 """Configuration state. Contains tuples where first item in a tuple is 

106 a method and remaining items are arguments for the method. 

107 """ 

108 

109 _initialized = False 

110 _componentSettings: list[ComponentSettings] = [] 

111 

112 _fileHandlers: list[logging.FileHandler] = [] 

113 """Any FileHandler classes attached to the root logger by this class 

114 that need to be closed on reset.""" 

115 

116 @staticmethod 

117 def root_loggers() -> set[str]: 

118 """Return the default root logger. 

119 

120 Returns 

121 ------- 

122 log_name : `set` of `str` 

123 The name(s) of the root logger(s) to use when the log level is 

124 being set without a log name being specified. 

125 

126 Notes 

127 ----- 

128 The default is ``lsst`` (which controls the butler infrastructure) 

129 but additional loggers can be specified by setting the environment 

130 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain 

131 multiple default loggers separated by a ``:``. 

132 """ 

133 log_names = {"lsst"} 

134 envvar = "DAF_BUTLER_ROOT_LOGGER" 

135 if envvar in os.environ: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true

136 log_names |= set(os.environ[envvar].split(":")) 

137 return log_names 

138 

139 @classmethod 

140 def initLog( 

141 cls, 

142 longlog: bool, 

143 log_tty: bool = True, 

144 log_file: tuple[str, ...] = (), 

145 log_label: dict[str, str] | None = None, 

146 ) -> None: 

147 """Initialize logging. This should only be called once per program 

148 execution. After the first call this will log a warning and return. 

149 

150 If lsst.log is importable, will add its log handler to the python 

151 root logger's handlers. 

152 

153 Parameters 

154 ---------- 

155 longlog : `bool` 

156 If True, make log messages appear in long format, by default False. 

157 log_tty : `bool` 

158 Control whether a default stream handler is enabled that logs 

159 to the terminal. 

160 log_file : `tuple` of `str` 

161 Path to files to write log records. If path ends in ``.json`` the 

162 records will be written in JSON format. Else they will be written 

163 in text format. If empty no log file will be created. Records 

164 will be appended to this file if it exists. 

165 log_label : `dict` of `str` 

166 Keys and values to be stored in logging MDC for all JSON log 

167 records. Keys will be upper-cased. 

168 """ 

169 if cls._initialized: 

170 # Unit tests that execute more than one command do end up 

171 # calling this function multiple times in one program execution, 

172 # so do log a debug but don't log an error or fail, just make the 

173 # re-initialization a no-op. 

174 log = logging.getLogger(__name__) 

175 log.debug("Log is already initialized, returning without re-initializing.") 

176 return 

177 cls._initialized = True 

178 cls._recordComponentSetting(None) 

179 

180 if lsstLog is not None: 

181 # Ensure that log messages are forwarded back to python. 

182 # Disable use of lsst.log MDC -- we expect butler uses to 

183 # use ButlerMDC. 

184 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

185 

186 # Forward python lsst.log messages directly to python logging. 

187 # This can bypass the C++ layer entirely but requires that 

188 # MDC is set via ButlerMDC, rather than in lsst.log. 

189 lsstLog.usePythonLogging() 

190 

191 formatter: logging.Formatter 

192 if not log_tty: 

193 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

194 elif longlog: 

195 # Want to create our own Formatter so that we can get high 

196 # precision timestamps. This requires we attach our own 

197 # default stream handler. 

198 defaultHandler = logging.StreamHandler() 

199 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

200 defaultHandler.setFormatter(formatter) 

201 

202 logging.basicConfig( 

203 level=logging.WARNING, 

204 force=True, 

205 handlers=[defaultHandler], 

206 ) 

207 

208 else: 

209 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{") 

210 

211 # Initialize the root logger. Calling this ensures that both 

212 # python loggers and lsst loggers are consistent in their default 

213 # logging level. 

214 cls._setLogLevel(".", "WARNING") 

215 

216 # Initialize default root logger level. 

217 cls._setLogLevel(None, "INFO") 

218 

219 # also capture warnings and send them to logging 

220 logging.captureWarnings(True) 

221 

222 # Create a record factory that ensures that an MDC is attached 

223 # to the records. By default this is only used for long-log 

224 # but always enable it for when someone adds a new handler 

225 # that needs it. 

226 ButlerMDC.add_mdc_log_record_factory() 

227 

228 # Set up the file logger 

229 for file in log_file: 

230 handler = logging.FileHandler(file) 

231 if file.endswith(".json"): 

232 formatter = JsonLogFormatter() 

233 else: 

234 if longlog: 

235 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

236 else: 

237 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

238 handler.setFormatter(formatter) 

239 logging.getLogger().addHandler(handler) 

240 cls._fileHandlers.append(handler) 

241 

242 # Add any requested MDC records. 

243 if log_label: 

244 for key, value in log_label.items(): 

245 ButlerMDC.MDC(key.upper(), value) 

246 

247 # remember this call 

248 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

249 

250 @classmethod 

251 def resetLog(cls) -> None: 

252 """Uninitialize the butler CLI Log handler and reset component log 

253 levels. 

254 

255 If the lsst.log handler was added to the python root logger's handlers 

256 in `initLog`, it will be removed here. 

257 

258 For each logger level that was set by this class, sets that logger's 

259 level to the value it was before this class set it. For lsst.log, if a 

260 component level was uninitialized, it will be set to 

261 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

262 component back to an uninitialized state. 

263 """ 

264 if lsstLog: 

265 lsstLog.doNotUsePythonLogging() 

266 for componentSetting in reversed(cls._componentSettings): 

267 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

268 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

269 logger = logging.getLogger(componentSetting.component) 

270 logger.setLevel(componentSetting.pythonLogLevel) 

271 cls._setLogLevel(None, "INFO") 

272 

273 ButlerMDC.restore_log_record_factory() 

274 

275 # Remove the FileHandler we may have attached. 

276 root = logging.getLogger() 

277 for handler in cls._fileHandlers: 

278 handler.close() 

279 root.removeHandler(handler) 

280 

281 cls._fileHandlers.clear() 

282 cls._initialized = False 

283 cls.configState = [] 

284 

285 @classmethod 

286 def setLogLevels(cls, logLevels: list[tuple[str | None, str]] | dict[str, str]) -> None: 

287 """Set log level for one or more components or the root logger. 

288 

289 Parameters 

290 ---------- 

291 logLevels : `list` of `tuple` 

292 Per-component logging levels, each item in the list is a tuple 

293 (component, level), `component` is a logger name or an empty string 

294 or `None` for default root logger, `level` is a logging level name, 

295 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

296 

297 Notes 

298 ----- 

299 The special name ``.`` can be used to set the Python root 

300 logger. 

301 """ 

302 if isinstance(logLevels, dict): 

303 logLevels = list(logLevels.items()) 

304 

305 # configure individual loggers 

306 for component, level in logLevels: 

307 cls._setLogLevel(component, level) 

308 # remember this call 

309 cls.configState.append((cls._setLogLevel, component, level)) 

310 

311 @classmethod 

312 def _setLogLevel(cls, component: str | None, level: str) -> None: 

313 """Set the log level for the given component. Record the current log 

314 level of the component so that it can be restored when resetting this 

315 log. 

316 

317 Parameters 

318 ---------- 

319 component : `str` or None 

320 The name of the log component or None for the default logger. 

321 The root logger can be specified either by an empty string or 

322 with the special name ``.``. 

323 level : `str` 

324 A valid python logging level. 

325 """ 

326 components: set[str | None] 

327 if component is None: 

328 components = set(cls.root_loggers()) 

329 elif not component or component == ".": 

330 components = {None} 

331 else: 

332 components = {component} 

333 for component in components: 

334 cls._recordComponentSetting(component) 

335 if lsstLog is not None: 

336 lsstLogger = lsstLog.Log.getLogger(component or "") 

337 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

338 pylevel = cls._getPyLogLevel(level) 

339 if pylevel is not None: 

340 logging.getLogger(component or None).setLevel(pylevel) 

341 

342 @staticmethod 

343 def _getPyLogLevel(level: str) -> int | None: 

344 """Get the numeric value for the given log level name. 

345 

346 Parameters 

347 ---------- 

348 level : `str` 

349 One of the python `logging` log level names. 

350 

351 Returns 

352 ------- 

353 numericValue : `int` 

354 The python `logging` numeric value for the log level. 

355 """ 

356 if level == "VERBOSE": 

357 return VERBOSE 

358 elif level == "TRACE": 

359 return TRACE 

360 return getattr(logging, level, None) 

361 

362 @staticmethod 

363 def _getLsstLogLevel(level: str) -> int | None: 

364 """Get the numeric value for the given log level name. 

365 

366 If `lsst.log` is not setup this function will return `None` regardless 

367 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

368 will not be setup when `daf_butler` is setup. Packages that depend on 

369 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

370 

371 Parameters 

372 ---------- 

373 level : `str` 

374 One of the python `logging` log level names. 

375 

376 Returns 

377 ------- 

378 numericValue : `int` or `None` 

379 The `lsst.log` numeric value. 

380 

381 Notes 

382 ----- 

383 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger. 

384 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be 

385 converted to ``DEBUG``. 

386 """ 

387 if lsstLog is None: 

388 return None 

389 if level == "VERBOSE": 

390 level = "INFO" 

391 elif level == "TRACE": 

392 level = "DEBUG" 

393 pylog_level = CliLog._getPyLogLevel(level) 

394 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

395 

396 class ComponentSettings: 

397 """Container for log level values for a logging component. 

398 

399 Parameters 

400 ---------- 

401 component : `str` or `None` 

402 The logger component. 

403 """ 

404 

405 def __init__(self, component: str | None): 

406 self.component = component 

407 self.pythonLogLevel = logging.getLogger(component).level 

408 self.lsstLogLevel = ( 

409 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

410 ) 

411 if self.lsstLogLevel == -1: 

412 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

413 

414 def __repr__(self) -> str: 

415 return ( 

416 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

417 f"lsstLogLevel={self.lsstLogLevel})" 

418 ) 

419 

420 @classmethod 

421 def _recordComponentSetting(cls, component: str | None) -> None: 

422 """Cache current levels for the given component in the list of 

423 component levels. 

424 """ 

425 componentSettings = cls.ComponentSettings(component) 

426 cls._componentSettings.append(componentSettings) 

427 

428 @classmethod 

429 def replayConfigState(cls, configState: list[tuple[Any, ...]]) -> None: 

430 """Re-create configuration using configuration state recorded earlier. 

431 

432 Parameters 

433 ---------- 

434 configState : `list` of `tuple` 

435 Tuples contain a method as first item and arguments for the method, 

436 in the same format as ``cls.configState``. 

437 """ 

438 if cls._initialized or cls.configState: 

439 # Already initialized, do not touch anything. 

440 log = logging.getLogger(__name__) 

441 log.warning("Log is already initialized, will not replay configuration.") 

442 return 

443 

444 # execute each one in order 

445 for call in configState: 

446 method, *args = call 

447 method(*args)