Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%

159 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-22 02:18 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "PrecisionLogFormatter", 

25 "CliLog", 

26) 

27 

28import datetime 

29import logging 

30import os 

31from typing import Any 

32 

33try: 

34 import lsst.log as lsstLog 

35except ModuleNotFoundError: 

36 lsstLog = None 

37 

38from lsst.utils.logging import TRACE, VERBOSE 

39 

40from ..core.logging import ButlerMDC, JsonLogFormatter 

41 

42 

43class PrecisionLogFormatter(logging.Formatter): 

44 """A log formatter that issues accurate timezone-aware timestamps.""" 

45 

46 converter = datetime.datetime.fromtimestamp # type: ignore 

47 

48 use_local = True 

49 """Control whether local time is displayed instead of UTC.""" 

50 

51 def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: 

52 """Format the time as an aware datetime.""" 

53 ct: datetime.datetime = self.converter(record.created, tz=datetime.timezone.utc) # type: ignore 

54 if self.use_local: 

55 ct = ct.astimezone() 

56 if datefmt: 

57 s = ct.strftime(datefmt) 

58 else: 

59 s = ct.isoformat(sep="T", timespec="milliseconds") 

60 return s 

61 

62 

63class CliLog: 

64 """Interface for managing python logging and ``lsst.log``. 

65 

66 This class defines log format strings for the log output and timestamp 

67 formats. It also configures ``lsst.log`` to forward all messages to 

68 Python `logging`. 

69 

70 This class can perform log uninitialization, which allows command line 

71 interface code that initializes logging to run unit tests that execute in 

72 batches, without affecting other unit tests. See ``resetLog``.""" 

73 

74 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

75 

76 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

77 """The log format used when the lsst.log package is not importable and the 

78 log is initialized with longlog=True.""" 

79 

80 pylog_normalFmt = "{name} {levelname}: {message}" 

81 """The log format used when the lsst.log package is not importable and the 

82 log is initialized with longlog=False.""" 

83 

84 configState: list[tuple[Any, ...]] = [] 

85 """Configuration state. Contains tuples where first item in a tuple is 

86 a method and remaining items are arguments for the method. 

87 """ 

88 

89 _initialized = False 

90 _componentSettings: list[ComponentSettings] = [] 

91 

92 _fileHandlers: list[logging.FileHandler] = [] 

93 """Any FileHandler classes attached to the root logger by this class 

94 that need to be closed on reset.""" 

95 

96 @staticmethod 

97 def root_loggers() -> set[str]: 

98 """Return the default root logger. 

99 

100 Returns 

101 ------- 

102 log_name : `set` of `str` 

103 The name(s) of the root logger(s) to use when the log level is 

104 being set without a log name being specified. 

105 

106 Notes 

107 ----- 

108 The default is ``lsst`` (which controls the butler infrastructure) 

109 but additional loggers can be specified by setting the environment 

110 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain 

111 multiple default loggers separated by a ``:``. 

112 """ 

113 log_names = set(["lsst"]) 

114 envvar = "DAF_BUTLER_ROOT_LOGGER" 

115 if envvar in os.environ: 115 ↛ 116line 115 didn't jump to line 116, because the condition on line 115 was never true

116 log_names |= set(os.environ[envvar].split(":")) 

117 return log_names 

118 

119 @classmethod 

120 def initLog( 

121 cls, 

122 longlog: bool, 

123 log_tty: bool = True, 

124 log_file: tuple[str, ...] = (), 

125 log_label: dict[str, str] | None = None, 

126 ) -> None: 

127 """Initialize logging. This should only be called once per program 

128 execution. After the first call this will log a warning and return. 

129 

130 If lsst.log is importable, will add its log handler to the python 

131 root logger's handlers. 

132 

133 Parameters 

134 ---------- 

135 longlog : `bool` 

136 If True, make log messages appear in long format, by default False. 

137 log_tty : `bool` 

138 Control whether a default stream handler is enabled that logs 

139 to the terminal. 

140 log_file : `tuple` of `str` 

141 Path to files to write log records. If path ends in ``.json`` the 

142 records will be written in JSON format. Else they will be written 

143 in text format. If empty no log file will be created. Records 

144 will be appended to this file if it exists. 

145 log_label : `dict` of `str` 

146 Keys and values to be stored in logging MDC for all JSON log 

147 records. Keys will be upper-cased. 

148 """ 

149 if cls._initialized: 

150 # Unit tests that execute more than one command do end up 

151 # calling this function multiple times in one program execution, 

152 # so do log a debug but don't log an error or fail, just make the 

153 # re-initialization a no-op. 

154 log = logging.getLogger(__name__) 

155 log.debug("Log is already initialized, returning without re-initializing.") 

156 return 

157 cls._initialized = True 

158 cls._recordComponentSetting(None) 

159 

160 if lsstLog is not None: 

161 # Ensure that log messages are forwarded back to python. 

162 # Disable use of lsst.log MDC -- we expect butler uses to 

163 # use ButlerMDC. 

164 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

165 

166 # Forward python lsst.log messages directly to python logging. 

167 # This can bypass the C++ layer entirely but requires that 

168 # MDC is set via ButlerMDC, rather than in lsst.log. 

169 lsstLog.usePythonLogging() 

170 

171 formatter: logging.Formatter 

172 if not log_tty: 

173 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

174 elif longlog: 

175 # Want to create our own Formatter so that we can get high 

176 # precision timestamps. This requires we attach our own 

177 # default stream handler. 

178 defaultHandler = logging.StreamHandler() 

179 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

180 defaultHandler.setFormatter(formatter) 

181 

182 logging.basicConfig( 

183 level=logging.WARNING, 

184 force=True, 

185 handlers=[defaultHandler], 

186 ) 

187 

188 else: 

189 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{") 

190 

191 # Initialize the root logger. Calling this ensures that both 

192 # python loggers and lsst loggers are consistent in their default 

193 # logging level. 

194 cls._setLogLevel(".", "WARNING") 

195 

196 # Initialize default root logger level. 

197 cls._setLogLevel(None, "INFO") 

198 

199 # also capture warnings and send them to logging 

200 logging.captureWarnings(True) 

201 

202 # Create a record factory that ensures that an MDC is attached 

203 # to the records. By default this is only used for long-log 

204 # but always enable it for when someone adds a new handler 

205 # that needs it. 

206 ButlerMDC.add_mdc_log_record_factory() 

207 

208 # Set up the file logger 

209 for file in log_file: 

210 handler = logging.FileHandler(file) 

211 if file.endswith(".json"): 

212 formatter = JsonLogFormatter() 

213 else: 

214 if longlog: 

215 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

216 else: 

217 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

218 handler.setFormatter(formatter) 

219 logging.getLogger().addHandler(handler) 

220 cls._fileHandlers.append(handler) 

221 

222 # Add any requested MDC records. 

223 if log_label: 

224 for key, value in log_label.items(): 

225 ButlerMDC.MDC(key.upper(), value) 

226 

227 # remember this call 

228 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

229 

230 @classmethod 

231 def resetLog(cls) -> None: 

232 """Uninitialize the butler CLI Log handler and reset component log 

233 levels. 

234 

235 If the lsst.log handler was added to the python root logger's handlers 

236 in `initLog`, it will be removed here. 

237 

238 For each logger level that was set by this class, sets that logger's 

239 level to the value it was before this class set it. For lsst.log, if a 

240 component level was uninitialized, it will be set to 

241 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

242 component back to an uninitialized state. 

243 """ 

244 if lsstLog: 

245 lsstLog.doNotUsePythonLogging() 

246 for componentSetting in reversed(cls._componentSettings): 

247 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

248 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

249 logger = logging.getLogger(componentSetting.component) 

250 logger.setLevel(componentSetting.pythonLogLevel) 

251 cls._setLogLevel(None, "INFO") 

252 

253 ButlerMDC.restore_log_record_factory() 

254 

255 # Remove the FileHandler we may have attached. 

256 root = logging.getLogger() 

257 for handler in cls._fileHandlers: 

258 handler.close() 

259 root.removeHandler(handler) 

260 

261 cls._fileHandlers.clear() 

262 cls._initialized = False 

263 cls.configState = [] 

264 

265 @classmethod 

266 def setLogLevels(cls, logLevels: list[tuple[str | None, str]] | dict[str, str]) -> None: 

267 """Set log level for one or more components or the root logger. 

268 

269 Parameters 

270 ---------- 

271 logLevels : `list` of `tuple` 

272 per-component logging levels, each item in the list is a tuple 

273 (component, level), `component` is a logger name or an empty string 

274 or `None` for default root logger, `level` is a logging level name, 

275 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

276 

277 Notes 

278 ----- 

279 The special name ``.`` can be used to set the Python root 

280 logger. 

281 """ 

282 if isinstance(logLevels, dict): 

283 logLevels = list(logLevels.items()) 

284 

285 # configure individual loggers 

286 for component, level in logLevels: 

287 cls._setLogLevel(component, level) 

288 # remember this call 

289 cls.configState.append((cls._setLogLevel, component, level)) 

290 

291 @classmethod 

292 def _setLogLevel(cls, component: str | None, level: str) -> None: 

293 """Set the log level for the given component. Record the current log 

294 level of the component so that it can be restored when resetting this 

295 log. 

296 

297 Parameters 

298 ---------- 

299 component : `str` or None 

300 The name of the log component or None for the default logger. 

301 The root logger can be specified either by an empty string or 

302 with the special name ``.``. 

303 level : `str` 

304 A valid python logging level. 

305 """ 

306 components: set[str | None] 

307 if component is None: 

308 components = {comp for comp in cls.root_loggers()} 

309 elif not component or component == ".": 

310 components = {None} 

311 else: 

312 components = {component} 

313 for component in components: 

314 cls._recordComponentSetting(component) 

315 if lsstLog is not None: 

316 lsstLogger = lsstLog.Log.getLogger(component or "") 

317 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

318 pylevel = cls._getPyLogLevel(level) 

319 if pylevel is not None: 

320 logging.getLogger(component or None).setLevel(pylevel) 

321 

322 @staticmethod 

323 def _getPyLogLevel(level: str) -> int | None: 

324 """Get the numeric value for the given log level name. 

325 

326 Parameters 

327 ---------- 

328 level : `str` 

329 One of the python `logging` log level names. 

330 

331 Returns 

332 ------- 

333 numericValue : `int` 

334 The python `logging` numeric value for the log level. 

335 """ 

336 if level == "VERBOSE": 

337 return VERBOSE 

338 elif level == "TRACE": 

339 return TRACE 

340 return getattr(logging, level, None) 

341 

342 @staticmethod 

343 def _getLsstLogLevel(level: str) -> int | None: 

344 """Get the numeric value for the given log level name. 

345 

346 If `lsst.log` is not setup this function will return `None` regardless 

347 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

348 will not be setup when `daf_butler` is setup. Packages that depend on 

349 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

350 

351 Parameters 

352 ---------- 

353 level : `str` 

354 One of the python `logging` log level names. 

355 

356 Returns 

357 ------- 

358 numericValue : `int` or `None` 

359 The `lsst.log` numeric value. 

360 

361 Notes 

362 ----- 

363 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger. 

364 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be 

365 converted to ``DEBUG``. 

366 """ 

367 if lsstLog is None: 

368 return None 

369 if level == "VERBOSE": 

370 level = "INFO" 

371 elif level == "TRACE": 

372 level = "DEBUG" 

373 pylog_level = CliLog._getPyLogLevel(level) 

374 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

375 

376 class ComponentSettings: 

377 """Container for log level values for a logging component.""" 

378 

379 def __init__(self, component: str | None): 

380 self.component = component 

381 self.pythonLogLevel = logging.getLogger(component).level 

382 self.lsstLogLevel = ( 

383 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

384 ) 

385 if self.lsstLogLevel == -1: 

386 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

387 

388 def __repr__(self) -> str: 

389 return ( 

390 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

391 f"lsstLogLevel={self.lsstLogLevel})" 

392 ) 

393 

394 @classmethod 

395 def _recordComponentSetting(cls, component: str | None) -> None: 

396 """Cache current levels for the given component in the list of 

397 component levels.""" 

398 componentSettings = cls.ComponentSettings(component) 

399 cls._componentSettings.append(componentSettings) 

400 

401 @classmethod 

402 def replayConfigState(cls, configState: list[tuple[Any, ...]]) -> None: 

403 """Re-create configuration using configuration state recorded earlier. 

404 

405 Parameters 

406 ---------- 

407 configState : `list` of `tuple` 

408 Tuples contain a method as first item and arguments for the method, 

409 in the same format as ``cls.configState``. 

410 """ 

411 if cls._initialized or cls.configState: 

412 # Already initialized, do not touch anything. 

413 log = logging.getLogger(__name__) 

414 log.warning("Log is already initialized, will not replay configuration.") 

415 return 

416 

417 # execute each one in order 

418 for call in configState: 

419 method, *args = call 

420 method(*args)