Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%

156 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-21 02:03 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23__all__ = ( 

24 "PrecisionLogFormatter", 

25 "CliLog", 

26) 

27 

28 

29import datetime 

30import logging 

31import os 

32from typing import Dict, Optional, Set, Tuple 

33 

34try: 

35 import lsst.log as lsstLog 

36except ModuleNotFoundError: 

37 lsstLog = None 

38 

39from lsst.utils.logging import TRACE, VERBOSE 

40 

41from ..core.logging import ButlerMDC, JsonLogFormatter 

42 

43 

44class PrecisionLogFormatter(logging.Formatter): 

45 """A log formatter that issues accurate timezone-aware timestamps.""" 

46 

47 converter = datetime.datetime.fromtimestamp 

48 

49 use_local = True 

50 """Control whether local time is displayed instead of UTC.""" 

51 

52 def formatTime(self, record, datefmt=None): 

53 """Format the time as an aware datetime.""" 

54 ct = self.converter(record.created, tz=datetime.timezone.utc) 

55 if self.use_local: 

56 ct = ct.astimezone() 

57 if datefmt: 

58 s = ct.strftime(datefmt) 

59 else: 

60 s = ct.isoformat(sep="T", timespec="milliseconds") 

61 return s 

62 

63 

64class CliLog: 

65 """Interface for managing python logging and ``lsst.log``. 

66 

67 This class defines log format strings for the log output and timestamp 

68 formats. It also configures ``lsst.log`` to forward all messages to 

69 Python `logging`. 

70 

71 This class can perform log uninitialization, which allows command line 

72 interface code that initializes logging to run unit tests that execute in 

73 batches, without affecting other unit tests. See ``resetLog``.""" 

74 

75 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

76 

77 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

78 """The log format used when the lsst.log package is not importable and the 

79 log is initialized with longlog=True.""" 

80 

81 pylog_normalFmt = "{name} {levelname}: {message}" 

82 """The log format used when the lsst.log package is not importable and the 

83 log is initialized with longlog=False.""" 

84 

85 configState = [] 

86 """Configuration state. Contains tuples where first item in a tuple is 

87 a method and remaining items are arguments for the method. 

88 """ 

89 

90 _initialized = False 

91 _componentSettings = [] 

92 

93 _fileHandlers = [] 

94 """Any FileHandler classes attached to the root logger by this class 

95 that need to be closed on reset.""" 

96 

97 @staticmethod 

98 def root_loggers() -> Set[str]: 

99 """Return the default root logger. 

100 

101 Returns 

102 ------- 

103 log_name : `set` of `str` 

104 The name(s) of the root logger(s) to use when the log level is 

105 being set without a log name being specified. 

106 

107 Notes 

108 ----- 

109 The default is ``lsst`` (which controls the butler infrastructure) 

110 but additional loggers can be specified by setting the environment 

111 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain 

112 multiple default loggers separated by a ``:``. 

113 """ 

114 log_names = set(["lsst"]) 

115 envvar = "DAF_BUTLER_ROOT_LOGGER" 

116 if envvar in os.environ: 116 ↛ 117line 116 didn't jump to line 117, because the condition on line 116 was never true

117 log_names |= set(os.environ[envvar].split(":")) 

118 return log_names 

119 

120 @classmethod 

121 def initLog( 

122 cls, 

123 longlog: bool, 

124 log_tty: bool = True, 

125 log_file: Tuple[str, ...] = (), 

126 log_label: Optional[Dict[str, str]] = None, 

127 ): 

128 """Initialize logging. This should only be called once per program 

129 execution. After the first call this will log a warning and return. 

130 

131 If lsst.log is importable, will add its log handler to the python 

132 root logger's handlers. 

133 

134 Parameters 

135 ---------- 

136 longlog : `bool` 

137 If True, make log messages appear in long format, by default False. 

138 log_tty : `bool` 

139 Control whether a default stream handler is enabled that logs 

140 to the terminal. 

141 log_file : `tuple` of `str` 

142 Path to files to write log records. If path ends in ``.json`` the 

143 records will be written in JSON format. Else they will be written 

144 in text format. If empty no log file will be created. Records 

145 will be appended to this file if it exists. 

146 log_label : `dict` of `str` 

147 Keys and values to be stored in logging MDC for all JSON log 

148 records. Keys will be upper-cased. 

149 """ 

150 if cls._initialized: 

151 # Unit tests that execute more than one command do end up 

152 # calling this function multiple times in one program execution, 

153 # so do log a debug but don't log an error or fail, just make the 

154 # re-initialization a no-op. 

155 log = logging.getLogger(__name__) 

156 log.debug("Log is already initialized, returning without re-initializing.") 

157 return 

158 cls._initialized = True 

159 cls._recordComponentSetting(None) 

160 

161 if lsstLog is not None: 

162 # Ensure that log messages are forwarded back to python. 

163 # Disable use of lsst.log MDC -- we expect butler uses to 

164 # use ButlerMDC. 

165 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

166 

167 # Forward python lsst.log messages directly to python logging. 

168 # This can bypass the C++ layer entirely but requires that 

169 # MDC is set via ButlerMDC, rather than in lsst.log. 

170 lsstLog.usePythonLogging() 

171 

172 if not log_tty: 

173 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

174 elif longlog: 

175 

176 # Want to create our own Formatter so that we can get high 

177 # precision timestamps. This requires we attach our own 

178 # default stream handler. 

179 defaultHandler = logging.StreamHandler() 

180 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

181 defaultHandler.setFormatter(formatter) 

182 

183 logging.basicConfig( 

184 level=logging.WARNING, 

185 force=True, 

186 handlers=[defaultHandler], 

187 ) 

188 

189 else: 

190 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{") 

191 

192 # Initialize the root logger. Calling this ensures that both 

193 # python loggers and lsst loggers are consistent in their default 

194 # logging level. 

195 cls._setLogLevel(".", "WARNING") 

196 

197 # Initialize default root logger level. 

198 cls._setLogLevel(None, "INFO") 

199 

200 # also capture warnings and send them to logging 

201 logging.captureWarnings(True) 

202 

203 # Create a record factory that ensures that an MDC is attached 

204 # to the records. By default this is only used for long-log 

205 # but always enable it for when someone adds a new handler 

206 # that needs it. 

207 ButlerMDC.add_mdc_log_record_factory() 

208 

209 # Set up the file logger 

210 for file in log_file: 

211 handler = logging.FileHandler(file) 

212 if file.endswith(".json"): 

213 formatter = JsonLogFormatter() 

214 else: 

215 if longlog: 

216 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

217 else: 

218 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

219 handler.setFormatter(formatter) 

220 logging.getLogger().addHandler(handler) 

221 cls._fileHandlers.append(handler) 

222 

223 # Add any requested MDC records. 

224 if log_label: 

225 for key, value in log_label.items(): 

226 ButlerMDC.MDC(key.upper(), value) 

227 

228 # remember this call 

229 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

230 

231 @classmethod 

232 def resetLog(cls): 

233 """Uninitialize the butler CLI Log handler and reset component log 

234 levels. 

235 

236 If the lsst.log handler was added to the python root logger's handlers 

237 in `initLog`, it will be removed here. 

238 

239 For each logger level that was set by this class, sets that logger's 

240 level to the value it was before this class set it. For lsst.log, if a 

241 component level was uninitialized, it will be set to 

242 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

243 component back to an uninitialized state. 

244 """ 

245 if lsstLog: 

246 lsstLog.doNotUsePythonLogging() 

247 for componentSetting in reversed(cls._componentSettings): 

248 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

249 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

250 logger = logging.getLogger(componentSetting.component) 

251 logger.setLevel(componentSetting.pythonLogLevel) 

252 cls._setLogLevel(None, "INFO") 

253 

254 ButlerMDC.restore_log_record_factory() 

255 

256 # Remove the FileHandler we may have attached. 

257 root = logging.getLogger() 

258 for handler in cls._fileHandlers: 

259 handler.close() 

260 root.removeHandler(handler) 

261 

262 cls._fileHandlers.clear() 

263 cls._initialized = False 

264 cls.configState = [] 

265 

266 @classmethod 

267 def setLogLevels(cls, logLevels): 

268 """Set log level for one or more components or the root logger. 

269 

270 Parameters 

271 ---------- 

272 logLevels : `list` of `tuple` 

273 per-component logging levels, each item in the list is a tuple 

274 (component, level), `component` is a logger name or an empty string 

275 or `None` for default root logger, `level` is a logging level name, 

276 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

277 

278 Notes 

279 ----- 

280 The special name ``.`` can be used to set the Python root 

281 logger. 

282 """ 

283 if isinstance(logLevels, dict): 

284 logLevels = logLevels.items() 

285 

286 # configure individual loggers 

287 for component, level in logLevels: 

288 cls._setLogLevel(component, level) 

289 # remember this call 

290 cls.configState.append((cls._setLogLevel, component, level)) 

291 

292 @classmethod 

293 def _setLogLevel(cls, component, level): 

294 """Set the log level for the given component. Record the current log 

295 level of the component so that it can be restored when resetting this 

296 log. 

297 

298 Parameters 

299 ---------- 

300 component : `str` or None 

301 The name of the log component or None for the default logger. 

302 The root logger can be specified either by an empty string or 

303 with the special name ``.``. 

304 level : `str` 

305 A valid python logging level. 

306 """ 

307 components: Set[Optional[str]] 

308 if component is None: 

309 components = cls.root_loggers() 

310 elif not component or component == ".": 

311 components = {None} 

312 else: 

313 components = {component} 

314 for component in components: 

315 cls._recordComponentSetting(component) 

316 if lsstLog is not None: 

317 lsstLogger = lsstLog.Log.getLogger(component or "") 

318 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

319 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level)) 

320 

321 @staticmethod 

322 def _getPyLogLevel(level): 

323 """Get the numeric value for the given log level name. 

324 

325 Parameters 

326 ---------- 

327 level : `str` 

328 One of the python `logging` log level names. 

329 

330 Returns 

331 ------- 

332 numericValue : `int` 

333 The python `logging` numeric value for the log level. 

334 """ 

335 if level == "VERBOSE": 

336 return VERBOSE 

337 elif level == "TRACE": 

338 return TRACE 

339 return getattr(logging, level, None) 

340 

341 @staticmethod 

342 def _getLsstLogLevel(level): 

343 """Get the numeric value for the given log level name. 

344 

345 If `lsst.log` is not setup this function will return `None` regardless 

346 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

347 will not be setup when `daf_butler` is setup. Packages that depend on 

348 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

349 

350 Parameters 

351 ---------- 

352 level : `str` 

353 One of the python `logging` log level names. 

354 

355 Returns 

356 ------- 

357 numericValue : `int` or `None` 

358 The `lsst.log` numeric value. 

359 

360 Notes 

361 ----- 

362 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger. 

363 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be 

364 converted to ``DEBUG``. 

365 """ 

366 if lsstLog is None: 

367 return None 

368 if level == "VERBOSE": 

369 level = "INFO" 

370 elif level == "TRACE": 

371 level = "DEBUG" 

372 pylog_level = CliLog._getPyLogLevel(level) 

373 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

374 

375 class ComponentSettings: 

376 """Container for log level values for a logging component.""" 

377 

378 def __init__(self, component): 

379 self.component = component 

380 self.pythonLogLevel = logging.getLogger(component).level 

381 self.lsstLogLevel = ( 

382 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

383 ) 

384 if self.lsstLogLevel == -1: 

385 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

386 

387 def __repr__(self): 

388 return ( 

389 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

390 f"lsstLogLevel={self.lsstLogLevel})" 

391 ) 

392 

393 @classmethod 

394 def _recordComponentSetting(cls, component): 

395 """Cache current levels for the given component in the list of 

396 component levels.""" 

397 componentSettings = cls.ComponentSettings(component) 

398 cls._componentSettings.append(componentSettings) 

399 

400 @classmethod 

401 def replayConfigState(cls, configState): 

402 """Re-create configuration using configuration state recorded earlier. 

403 

404 Parameters 

405 ---------- 

406 configState : `list` of `tuple` 

407 Tuples contain a method as first item and arguments for the method, 

408 in the same format as ``cls.configState``. 

409 """ 

410 if cls._initialized or cls.configState: 

411 # Already initialized, do not touch anything. 

412 log = logging.getLogger(__name__) 

413 log.warning("Log is already initialized, will not replay configuration.") 

414 return 

415 

416 # execute each one in order 

417 for call in configState: 

418 method, *args = call 

419 method(*args)