Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%

150 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-24 02:27 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import datetime 

23import logging 

24import os 

25from typing import Dict, Optional, Set, Tuple 

26 

27try: 

28 import lsst.log as lsstLog 

29except ModuleNotFoundError: 

30 lsstLog = None 

31 

32from lsst.utils.logging import TRACE, VERBOSE 

33 

34from ..core.logging import ButlerMDC, JsonLogFormatter 

35 

36 

37class PrecisionLogFormatter(logging.Formatter): 

38 """A log formatter that issues accurate timezone-aware timestamps.""" 

39 

40 converter = datetime.datetime.fromtimestamp 

41 

42 use_local = True 

43 """Control whether local time is displayed instead of UTC.""" 

44 

45 def formatTime(self, record, datefmt=None): 

46 """Format the time as an aware datetime.""" 

47 ct = self.converter(record.created, tz=datetime.timezone.utc) 

48 if self.use_local: 

49 ct = ct.astimezone() 

50 if datefmt: 

51 s = ct.strftime(datefmt) 

52 else: 

53 s = ct.isoformat(sep="T", timespec="milliseconds") 

54 return s 

55 

56 

57class CliLog: 

58 """Interface for managing python logging and ``lsst.log``. 

59 

60 This class defines log format strings for the log output and timestamp 

61 formats. It also configures ``lsst.log`` to forward all messages to 

62 Python `logging`. 

63 

64 This class can perform log uninitialization, which allows command line 

65 interface code that initializes logging to run unit tests that execute in 

66 batches, without affecting other unit tests. See ``resetLog``.""" 

67 

68 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

69 

70 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

71 """The log format used when the lsst.log package is not importable and the 

72 log is initialized with longlog=True.""" 

73 

74 pylog_normalFmt = "{name} {levelname}: {message}" 

75 """The log format used when the lsst.log package is not importable and the 

76 log is initialized with longlog=False.""" 

77 

78 configState = [] 

79 """Configuration state. Contains tuples where first item in a tuple is 

80 a method and remaining items are arguments for the method. 

81 """ 

82 

83 _initialized = False 

84 _componentSettings = [] 

85 

86 _fileHandlers = [] 

87 """Any FileHandler classes attached to the root logger by this class 

88 that need to be closed on reset.""" 

89 

90 @staticmethod 

91 def root_loggers() -> Set[str]: 

92 """Return the default root logger. 

93 

94 Returns 

95 ------- 

96 log_name : `set` of `str` 

97 The name(s) of the root logger(s) to use when the log level is 

98 being set without a log name being specified. 

99 

100 Notes 

101 ----- 

102 The default is ``lsst`` (which controls the butler infrastructure) 

103 but additional loggers can be specified by setting the environment 

104 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain 

105 multiple default loggers separated by a ``:``. 

106 """ 

107 log_names = set(["lsst"]) 

108 envvar = "DAF_BUTLER_ROOT_LOGGER" 

109 if envvar in os.environ: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true

110 log_names |= set(os.environ[envvar].split(":")) 

111 return log_names 

112 

113 @classmethod 

114 def initLog( 

115 cls, 

116 longlog: bool, 

117 log_tty: bool = True, 

118 log_file: Tuple[str, ...] = (), 

119 log_label: Optional[Dict[str, str]] = None, 

120 ): 

121 """Initialize logging. This should only be called once per program 

122 execution. After the first call this will log a warning and return. 

123 

124 If lsst.log is importable, will add its log handler to the python 

125 root logger's handlers. 

126 

127 Parameters 

128 ---------- 

129 longlog : `bool` 

130 If True, make log messages appear in long format, by default False. 

131 log_tty : `bool` 

132 Control whether a default stream handler is enabled that logs 

133 to the terminal. 

134 log_file : `tuple` of `str` 

135 Path to files to write log records. If path ends in ``.json`` the 

136 records will be written in JSON format. Else they will be written 

137 in text format. If empty no log file will be created. Records 

138 will be appended to this file if it exists. 

139 log_label : `dict` of `str` 

140 Keys and values to be stored in logging MDC for all JSON log 

141 records. Keys will be upper-cased. 

142 """ 

143 if cls._initialized: 

144 # Unit tests that execute more than one command do end up 

145 # calling this function multiple times in one program execution, 

146 # so do log a debug but don't log an error or fail, just make the 

147 # re-initialization a no-op. 

148 log = logging.getLogger(__name__) 

149 log.debug("Log is already initialized, returning without re-initializing.") 

150 return 

151 cls._initialized = True 

152 cls._recordComponentSetting(None) 

153 

154 if lsstLog is not None: 

155 # Ensure that log messages are forwarded back to python. 

156 # Disable use of lsst.log MDC -- we expect butler uses to 

157 # use ButlerMDC. 

158 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

159 

160 # Forward python lsst.log messages directly to python logging. 

161 # This can bypass the C++ layer entirely but requires that 

162 # MDC is set via ButlerMDC, rather than in lsst.log. 

163 lsstLog.usePythonLogging() 

164 

165 if not log_tty: 

166 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

167 elif longlog: 

168 

169 # Want to create our own Formatter so that we can get high 

170 # precision timestamps. This requires we attach our own 

171 # default stream handler. 

172 defaultHandler = logging.StreamHandler() 

173 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

174 defaultHandler.setFormatter(formatter) 

175 

176 logging.basicConfig( 

177 level=logging.WARNING, 

178 force=True, 

179 handlers=[defaultHandler], 

180 ) 

181 

182 else: 

183 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{") 

184 

185 # Initialize the root logger. Calling this ensures that both 

186 # python loggers and lsst loggers are consistent in their default 

187 # logging level. 

188 cls._setLogLevel(".", "WARNING") 

189 

190 # Initialize default root logger level. 

191 cls._setLogLevel(None, "INFO") 

192 

193 # also capture warnings and send them to logging 

194 logging.captureWarnings(True) 

195 

196 # Create a record factory that ensures that an MDC is attached 

197 # to the records. By default this is only used for long-log 

198 # but always enable it for when someone adds a new handler 

199 # that needs it. 

200 ButlerMDC.add_mdc_log_record_factory() 

201 

202 # Set up the file logger 

203 for file in log_file: 

204 handler = logging.FileHandler(file) 

205 if file.endswith(".json"): 

206 formatter = JsonLogFormatter() 

207 else: 

208 if longlog: 

209 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

210 else: 

211 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

212 handler.setFormatter(formatter) 

213 logging.getLogger().addHandler(handler) 

214 cls._fileHandlers.append(handler) 

215 

216 # Add any requested MDC records. 

217 if log_label: 

218 for key, value in log_label.items(): 

219 ButlerMDC.MDC(key.upper(), value) 

220 

221 # remember this call 

222 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

223 

224 @classmethod 

225 def resetLog(cls): 

226 """Uninitialize the butler CLI Log handler and reset component log 

227 levels. 

228 

229 If the lsst.log handler was added to the python root logger's handlers 

230 in `initLog`, it will be removed here. 

231 

232 For each logger level that was set by this class, sets that logger's 

233 level to the value it was before this class set it. For lsst.log, if a 

234 component level was uninitialized, it will be set to 

235 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

236 component back to an uninitialized state. 

237 """ 

238 if lsstLog: 

239 lsstLog.doNotUsePythonLogging() 

240 for componentSetting in reversed(cls._componentSettings): 

241 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

242 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

243 logger = logging.getLogger(componentSetting.component) 

244 logger.setLevel(componentSetting.pythonLogLevel) 

245 cls._setLogLevel(None, "INFO") 

246 

247 ButlerMDC.restore_log_record_factory() 

248 

249 # Remove the FileHandler we may have attached. 

250 root = logging.getLogger() 

251 for handler in cls._fileHandlers: 

252 handler.close() 

253 root.removeHandler(handler) 

254 

255 cls._fileHandlers.clear() 

256 cls._initialized = False 

257 cls.configState = [] 

258 

259 @classmethod 

260 def setLogLevels(cls, logLevels): 

261 """Set log level for one or more components or the root logger. 

262 

263 Parameters 

264 ---------- 

265 logLevels : `list` of `tuple` 

266 per-component logging levels, each item in the list is a tuple 

267 (component, level), `component` is a logger name or an empty string 

268 or `None` for default root logger, `level` is a logging level name, 

269 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

270 

271 Notes 

272 ----- 

273 The special name ``.`` can be used to set the Python root 

274 logger. 

275 """ 

276 if isinstance(logLevels, dict): 

277 logLevels = logLevels.items() 

278 

279 # configure individual loggers 

280 for component, level in logLevels: 

281 cls._setLogLevel(component, level) 

282 # remember this call 

283 cls.configState.append((cls._setLogLevel, component, level)) 

284 

285 @classmethod 

286 def _setLogLevel(cls, component, level): 

287 """Set the log level for the given component. Record the current log 

288 level of the component so that it can be restored when resetting this 

289 log. 

290 

291 Parameters 

292 ---------- 

293 component : `str` or None 

294 The name of the log component or None for the default logger. 

295 The root logger can be specified either by an empty string or 

296 with the special name ``.``. 

297 level : `str` 

298 A valid python logging level. 

299 """ 

300 components: Set[Optional[str]] 

301 if component is None: 

302 components = cls.root_loggers() 

303 elif not component or component == ".": 

304 components = {None} 

305 else: 

306 components = {component} 

307 for component in components: 

308 cls._recordComponentSetting(component) 

309 if lsstLog is not None: 

310 lsstLogger = lsstLog.Log.getLogger(component or "") 

311 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

312 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level)) 

313 

314 @staticmethod 

315 def _getPyLogLevel(level): 

316 """Get the numeric value for the given log level name. 

317 

318 Parameters 

319 ---------- 

320 level : `str` 

321 One of the python `logging` log level names. 

322 

323 Returns 

324 ------- 

325 numericValue : `int` 

326 The python `logging` numeric value for the log level. 

327 """ 

328 if level == "VERBOSE": 

329 return VERBOSE 

330 elif level == "TRACE": 

331 return TRACE 

332 return getattr(logging, level, None) 

333 

334 @staticmethod 

335 def _getLsstLogLevel(level): 

336 """Get the numeric value for the given log level name. 

337 

338 If `lsst.log` is not setup this function will return `None` regardless 

339 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

340 will not be setup when `daf_butler` is setup. Packages that depend on 

341 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

342 

343 Parameters 

344 ---------- 

345 level : `str` 

346 One of the python `logging` log level names. 

347 

348 Returns 

349 ------- 

350 numericValue : `int` or `None` 

351 The `lsst.log` numeric value. 

352 

353 Notes 

354 ----- 

355 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger. 

356 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be 

357 converted to ``DEBUG``. 

358 """ 

359 if lsstLog is None: 

360 return None 

361 if level == "VERBOSE": 

362 level = "INFO" 

363 elif level == "TRACE": 

364 level = "DEBUG" 

365 pylog_level = CliLog._getPyLogLevel(level) 

366 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

367 

368 class ComponentSettings: 

369 """Container for log level values for a logging component.""" 

370 

371 def __init__(self, component): 

372 self.component = component 

373 self.pythonLogLevel = logging.getLogger(component).level 

374 self.lsstLogLevel = ( 

375 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

376 ) 

377 if self.lsstLogLevel == -1: 

378 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

379 

380 def __repr__(self): 

381 return ( 

382 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

383 f"lsstLogLevel={self.lsstLogLevel})" 

384 ) 

385 

386 @classmethod 

387 def _recordComponentSetting(cls, component): 

388 """Cache current levels for the given component in the list of 

389 component levels.""" 

390 componentSettings = cls.ComponentSettings(component) 

391 cls._componentSettings.append(componentSettings) 

392 

393 @classmethod 

394 def replayConfigState(cls, configState): 

395 """Re-create configuration using configuration state recorded earlier. 

396 

397 Parameters 

398 ---------- 

399 configState : `list` of `tuple` 

400 Tuples contain a method as first item and arguments for the method, 

401 in the same format as ``cls.configState``. 

402 """ 

403 if cls._initialized or cls.configState: 

404 # Already initialized, do not touch anything. 

405 log = logging.getLogger(__name__) 

406 log.warning("Log is already initialized, will not replay configuration.") 

407 return 

408 

409 # execute each one in order 

410 for call in configState: 

411 method, *args = call 

412 method(*args)