Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

131 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import datetime 

23import logging 

24from typing import Dict, Optional, Tuple 

25 

26try: 

27 import lsst.log as lsstLog 

28except ModuleNotFoundError: 

29 lsstLog = None 

30 

31from lsst.utils.logging import VERBOSE 

32 

33from ..core.logging import ButlerMDC, JsonLogFormatter 

34 

35 

36class PrecisionLogFormatter(logging.Formatter): 

37 """A log formatter that issues accurate timezone-aware timestamps.""" 

38 

39 converter = datetime.datetime.fromtimestamp 

40 

41 use_local = True 

42 """Control whether local time is displayed instead of UTC.""" 

43 

44 def formatTime(self, record, datefmt=None): 

45 """Format the time as an aware datetime.""" 

46 ct = self.converter(record.created, tz=datetime.timezone.utc) 

47 if self.use_local: 

48 ct = ct.astimezone() 

49 if datefmt: 

50 s = ct.strftime(datefmt) 

51 else: 

52 s = ct.isoformat(sep="T", timespec="milliseconds") 

53 return s 

54 

55 

56class CliLog: 

57 """Interface for managing python logging and ``lsst.log``. 

58 

59 This class defines log format strings for the log output and timestamp 

60 formats. It also configures ``lsst.log`` to forward all messages to 

61 Python `logging`. 

62 

63 This class can perform log uninitialization, which allows command line 

64 interface code that initializes logging to run unit tests that execute in 

65 batches, without affecting other unit tests. See ``resetLog``.""" 

66 

67 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

68 

69 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

70 """The log format used when the lsst.log package is not importable and the 

71 log is initialized with longlog=True.""" 

72 

73 pylog_normalFmt = "{name} {levelname}: {message}" 

74 """The log format used when the lsst.log package is not importable and the 

75 log is initialized with longlog=False.""" 

76 

77 configState = [] 

78 """Configuration state. Contains tuples where first item in a tuple is 

79 a method and remaining items are arguments for the method. 

80 """ 

81 

82 _initialized = False 

83 _componentSettings = [] 

84 

85 _fileHandlers = [] 

86 """Any FileHandler classes attached to the root logger by this class 

87 that need to be closed on reset.""" 

88 

89 @classmethod 

90 def initLog( 

91 cls, 

92 longlog: bool, 

93 log_tty: bool = True, 

94 log_file: Tuple[str, ...] = (), 

95 log_label: Optional[Dict[str, str]] = None, 

96 ): 

97 """Initialize logging. This should only be called once per program 

98 execution. After the first call this will log a warning and return. 

99 

100 If lsst.log is importable, will add its log handler to the python 

101 root logger's handlers. 

102 

103 Parameters 

104 ---------- 

105 longlog : `bool` 

106 If True, make log messages appear in long format, by default False. 

107 log_tty : `bool` 

108 Control whether a default stream handler is enabled that logs 

109 to the terminal. 

110 log_file : `tuple` of `str` 

111 Path to files to write log records. If path ends in ``.json`` the 

112 records will be written in JSON format. Else they will be written 

113 in text format. If empty no log file will be created. Records 

114 will be appended to this file if it exists. 

115 log_label : `dict` of `str` 

116 Keys and values to be stored in logging MDC for all JSON log 

117 records. Keys will be upper-cased. 

118 """ 

119 if cls._initialized: 

120 # Unit tests that execute more than one command do end up 

121 # calling this function multiple times in one program execution, 

122 # so do log a debug but don't log an error or fail, just make the 

123 # re-initialization a no-op. 

124 log = logging.getLogger(__name__) 

125 log.debug("Log is already initialized, returning without re-initializing.") 

126 return 

127 cls._initialized = True 

128 cls._recordComponentSetting(None) 

129 

130 if lsstLog is not None: 

131 # Ensure that log messages are forwarded back to python. 

132 # Disable use of lsst.log MDC -- we expect butler uses to 

133 # use ButlerMDC. 

134 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

135 

136 # Forward python lsst.log messages directly to python logging. 

137 # This can bypass the C++ layer entirely but requires that 

138 # MDC is set via ButlerMDC, rather than in lsst.log. 

139 lsstLog.usePythonLogging() 

140 

141 if not log_tty: 

142 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

143 elif longlog: 

144 

145 # Want to create our own Formatter so that we can get high 

146 # precision timestamps. This requires we attach our own 

147 # default stream handler. 

148 defaultHandler = logging.StreamHandler() 

149 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

150 defaultHandler.setFormatter(formatter) 

151 

152 logging.basicConfig( 

153 level=logging.INFO, 

154 force=True, 

155 handlers=[defaultHandler], 

156 ) 

157 

158 else: 

159 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{") 

160 

161 # Initialize root logger level. 

162 cls._setLogLevel(None, "INFO") 

163 

164 # also capture warnings and send them to logging 

165 logging.captureWarnings(True) 

166 

167 # Create a record factory that ensures that an MDC is attached 

168 # to the records. By default this is only used for long-log 

169 # but always enable it for when someone adds a new handler 

170 # that needs it. 

171 ButlerMDC.add_mdc_log_record_factory() 

172 

173 # Set up the file logger 

174 for file in log_file: 

175 handler = logging.FileHandler(file) 

176 if file.endswith(".json"): 

177 formatter = JsonLogFormatter() 

178 else: 

179 if longlog: 

180 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

181 else: 

182 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

183 handler.setFormatter(formatter) 

184 logging.getLogger().addHandler(handler) 

185 cls._fileHandlers.append(handler) 

186 

187 # Add any requested MDC records. 

188 if log_label: 

189 for key, value in log_label.items(): 

190 ButlerMDC.MDC(key.upper(), value) 

191 

192 # remember this call 

193 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

194 

195 @classmethod 

196 def resetLog(cls): 

197 """Uninitialize the butler CLI Log handler and reset component log 

198 levels. 

199 

200 If the lsst.log handler was added to the python root logger's handlers 

201 in `initLog`, it will be removed here. 

202 

203 For each logger level that was set by this class, sets that logger's 

204 level to the value it was before this class set it. For lsst.log, if a 

205 component level was uninitialized, it will be set to 

206 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

207 component back to an uninitialized state. 

208 """ 

209 if lsstLog: 

210 lsstLog.doNotUsePythonLogging() 

211 for componentSetting in reversed(cls._componentSettings): 

212 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

213 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

214 logger = logging.getLogger(componentSetting.component) 

215 logger.setLevel(componentSetting.pythonLogLevel) 

216 cls._setLogLevel(None, "INFO") 

217 

218 ButlerMDC.restore_log_record_factory() 

219 

220 # Remove the FileHandler we may have attached. 

221 root = logging.getLogger() 

222 for handler in cls._fileHandlers: 

223 handler.close() 

224 root.removeHandler(handler) 

225 

226 cls._fileHandlers.clear() 

227 cls._initialized = False 

228 cls.configState = [] 

229 

230 @classmethod 

231 def setLogLevels(cls, logLevels): 

232 """Set log level for one or more components or the root logger. 

233 

234 Parameters 

235 ---------- 

236 logLevels : `list` of `tuple` 

237 per-component logging levels, each item in the list is a tuple 

238 (component, level), `component` is a logger name or an empty string 

239 or `None` for root logger, `level` is a logging level name, one of 

240 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

241 """ 

242 if isinstance(logLevels, dict): 

243 logLevels = logLevels.items() 

244 

245 # configure individual loggers 

246 for component, level in logLevels: 

247 cls._setLogLevel(component, level) 

248 # remember this call 

249 cls.configState.append((cls._setLogLevel, component, level)) 

250 

251 @classmethod 

252 def _setLogLevel(cls, component, level): 

253 """Set the log level for the given component. Record the current log 

254 level of the component so that it can be restored when resetting this 

255 log. 

256 

257 Parameters 

258 ---------- 

259 component : `str` or None 

260 The name of the log component or None for the root logger. 

261 level : `str` 

262 A valid python logging level. 

263 """ 

264 cls._recordComponentSetting(component) 

265 if lsstLog is not None: 

266 lsstLogger = lsstLog.Log.getLogger(component or "") 

267 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

268 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level)) 

269 

270 @staticmethod 

271 def _getPyLogLevel(level): 

272 """Get the numeric value for the given log level name. 

273 

274 Parameters 

275 ---------- 

276 level : `str` 

277 One of the python `logging` log level names. 

278 

279 Returns 

280 ------- 

281 numericValue : `int` 

282 The python `logging` numeric value for the log level. 

283 """ 

284 if level == "VERBOSE": 

285 return VERBOSE 

286 return getattr(logging, level, None) 

287 

288 @staticmethod 

289 def _getLsstLogLevel(level): 

290 """Get the numeric value for the given log level name. 

291 

292 If `lsst.log` is not setup this function will return `None` regardless 

293 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

294 will not be setup when `daf_butler` is setup. Packages that depend on 

295 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

296 

297 Parameters 

298 ---------- 

299 level : `str` 

300 One of the python `logging` log level names. 

301 

302 Returns 

303 ------- 

304 numericValue : `int` or `None` 

305 The `lsst.log` numeric value. 

306 

307 Notes 

308 ----- 

309 ``VERBOSE`` logging is not supported by the LSST logger and so will 

310 always be converted to ``INFO``. 

311 """ 

312 if lsstLog is None: 

313 return None 

314 if level == "VERBOSE": 

315 level = "INFO" 

316 pylog_level = CliLog._getPyLogLevel(level) 

317 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

318 

319 class ComponentSettings: 

320 """Container for log level values for a logging component.""" 

321 

322 def __init__(self, component): 

323 self.component = component 

324 self.pythonLogLevel = logging.getLogger(component).level 

325 self.lsstLogLevel = ( 

326 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

327 ) 

328 if self.lsstLogLevel == -1: 

329 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

330 

331 def __repr__(self): 

332 return ( 

333 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

334 f"lsstLogLevel={self.lsstLogLevel})" 

335 ) 

336 

337 @classmethod 

338 def _recordComponentSetting(cls, component): 

339 """Cache current levels for the given component in the list of 

340 component levels.""" 

341 componentSettings = cls.ComponentSettings(component) 

342 cls._componentSettings.append(componentSettings) 

343 

344 @classmethod 

345 def replayConfigState(cls, configState): 

346 """Re-create configuration using configuration state recorded earlier. 

347 

348 Parameters 

349 ---------- 

350 configState : `list` of `tuple` 

351 Tuples contain a method as first item and arguments for the method, 

352 in the same format as ``cls.configState``. 

353 """ 

354 if cls._initialized or cls.configState: 

355 # Already initialized, do not touch anything. 

356 log = logging.getLogger(__name__) 

357 log.warning("Log is already initialized, will not replay configuration.") 

358 return 

359 

360 # execute each one in order 

361 for call in configState: 

362 method, *args = call 

363 method(*args)