Coverage for python/lsst/daf/butler/cli/cliLog.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

149 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import datetime 

23import logging 

24import os 

25from typing import Dict, Optional, Set, Tuple 

26 

27try: 

28 import lsst.log as lsstLog 

29except ModuleNotFoundError: 

30 lsstLog = None 

31 

32from lsst.utils.logging import TRACE, VERBOSE 

33 

34from ..core.logging import ButlerMDC, JsonLogFormatter 

35 

36 

37class PrecisionLogFormatter(logging.Formatter): 

38 """A log formatter that issues accurate timezone-aware timestamps.""" 

39 

40 converter = datetime.datetime.fromtimestamp 

41 

42 use_local = True 

43 """Control whether local time is displayed instead of UTC.""" 

44 

45 def formatTime(self, record, datefmt=None): 

46 """Format the time as an aware datetime.""" 

47 ct = self.converter(record.created, tz=datetime.timezone.utc) 

48 if self.use_local: 

49 ct = ct.astimezone() 

50 if datefmt: 

51 s = ct.strftime(datefmt) 

52 else: 

53 s = ct.isoformat(sep="T", timespec="milliseconds") 

54 return s 

55 

56 

57class CliLog: 

58 """Interface for managing python logging and ``lsst.log``. 

59 

60 This class defines log format strings for the log output and timestamp 

61 formats. It also configures ``lsst.log`` to forward all messages to 

62 Python `logging`. 

63 

64 This class can perform log uninitialization, which allows command line 

65 interface code that initializes logging to run unit tests that execute in 

66 batches, without affecting other unit tests. See ``resetLog``.""" 

67 

68 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

69 

70 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

71 """The log format used when the lsst.log package is not importable and the 

72 log is initialized with longlog=True.""" 

73 

74 pylog_normalFmt = "{name} {levelname}: {message}" 

75 """The log format used when the lsst.log package is not importable and the 

76 log is initialized with longlog=False.""" 

77 

78 configState = [] 

79 """Configuration state. Contains tuples where first item in a tuple is 

80 a method and remaining items are arguments for the method. 

81 """ 

82 

83 _initialized = False 

84 _componentSettings = [] 

85 

86 _fileHandlers = [] 

87 """Any FileHandler classes attached to the root logger by this class 

88 that need to be closed on reset.""" 

89 

90 @staticmethod 

91 def root_loggers() -> Set[str]: 

92 """Return the default root logger. 

93 

94 Returns 

95 ------- 

96 log_name : `set` of `str` 

97 The name(s) of the root logger(s) to use when the log level is 

98 being set without a log name being specified. 

99 

100 Notes 

101 ----- 

102 The default is ``lsst`` (which controls the butler infrastructure) 

103 but additional loggers can be specified by setting the environment 

104 variable ``DAF_BUTLER_ROOT_LOGGER``. This variable can contain 

105 multiple default loggers separated by a ``:``. 

106 """ 

107 log_names = set(["lsst"]) 

108 envvar = "DAF_BUTLER_ROOT_LOGGER" 

109 if envvar in os.environ: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true

110 log_names |= set(os.environ[envvar].split(":")) 

111 return log_names 

112 

113 @classmethod 

114 def initLog( 

115 cls, 

116 longlog: bool, 

117 log_tty: bool = True, 

118 log_file: Tuple[str, ...] = (), 

119 log_label: Optional[Dict[str, str]] = None, 

120 ): 

121 """Initialize logging. This should only be called once per program 

122 execution. After the first call this will log a warning and return. 

123 

124 If lsst.log is importable, will add its log handler to the python 

125 root logger's handlers. 

126 

127 Parameters 

128 ---------- 

129 longlog : `bool` 

130 If True, make log messages appear in long format, by default False. 

131 log_tty : `bool` 

132 Control whether a default stream handler is enabled that logs 

133 to the terminal. 

134 log_file : `tuple` of `str` 

135 Path to files to write log records. If path ends in ``.json`` the 

136 records will be written in JSON format. Else they will be written 

137 in text format. If empty no log file will be created. Records 

138 will be appended to this file if it exists. 

139 log_label : `dict` of `str` 

140 Keys and values to be stored in logging MDC for all JSON log 

141 records. Keys will be upper-cased. 

142 """ 

143 if cls._initialized: 

144 # Unit tests that execute more than one command do end up 

145 # calling this function multiple times in one program execution, 

146 # so do log a debug but don't log an error or fail, just make the 

147 # re-initialization a no-op. 

148 log = logging.getLogger(__name__) 

149 log.debug("Log is already initialized, returning without re-initializing.") 

150 return 

151 cls._initialized = True 

152 cls._recordComponentSetting(None) 

153 

154 if lsstLog is not None: 

155 # Ensure that log messages are forwarded back to python. 

156 # Disable use of lsst.log MDC -- we expect butler uses to 

157 # use ButlerMDC. 

158 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

159 

160 # Forward python lsst.log messages directly to python logging. 

161 # This can bypass the C++ layer entirely but requires that 

162 # MDC is set via ButlerMDC, rather than in lsst.log. 

163 lsstLog.usePythonLogging() 

164 

165 if not log_tty: 

166 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

167 elif longlog: 

168 

169 # Want to create our own Formatter so that we can get high 

170 # precision timestamps. This requires we attach our own 

171 # default stream handler. 

172 defaultHandler = logging.StreamHandler() 

173 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

174 defaultHandler.setFormatter(formatter) 

175 

176 logging.basicConfig( 

177 level=logging.WARNING, 

178 force=True, 

179 handlers=[defaultHandler], 

180 ) 

181 

182 else: 

183 logging.basicConfig(level=logging.WARNING, format=cls.pylog_normalFmt, style="{") 

184 

185 # Initialize root logger level. 

186 cls._setLogLevel(None, "INFO") 

187 

188 # also capture warnings and send them to logging 

189 logging.captureWarnings(True) 

190 

191 # Create a record factory that ensures that an MDC is attached 

192 # to the records. By default this is only used for long-log 

193 # but always enable it for when someone adds a new handler 

194 # that needs it. 

195 ButlerMDC.add_mdc_log_record_factory() 

196 

197 # Set up the file logger 

198 for file in log_file: 

199 handler = logging.FileHandler(file) 

200 if file.endswith(".json"): 

201 formatter = JsonLogFormatter() 

202 else: 

203 if longlog: 

204 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

205 else: 

206 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

207 handler.setFormatter(formatter) 

208 logging.getLogger().addHandler(handler) 

209 cls._fileHandlers.append(handler) 

210 

211 # Add any requested MDC records. 

212 if log_label: 

213 for key, value in log_label.items(): 

214 ButlerMDC.MDC(key.upper(), value) 

215 

216 # remember this call 

217 cls.configState.append((cls.initLog, longlog, log_tty, log_file, log_label)) 

218 

219 @classmethod 

220 def resetLog(cls): 

221 """Uninitialize the butler CLI Log handler and reset component log 

222 levels. 

223 

224 If the lsst.log handler was added to the python root logger's handlers 

225 in `initLog`, it will be removed here. 

226 

227 For each logger level that was set by this class, sets that logger's 

228 level to the value it was before this class set it. For lsst.log, if a 

229 component level was uninitialized, it will be set to 

230 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

231 component back to an uninitialized state. 

232 """ 

233 if lsstLog: 

234 lsstLog.doNotUsePythonLogging() 

235 for componentSetting in reversed(cls._componentSettings): 

236 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

237 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

238 logger = logging.getLogger(componentSetting.component) 

239 logger.setLevel(componentSetting.pythonLogLevel) 

240 cls._setLogLevel(None, "INFO") 

241 

242 ButlerMDC.restore_log_record_factory() 

243 

244 # Remove the FileHandler we may have attached. 

245 root = logging.getLogger() 

246 for handler in cls._fileHandlers: 

247 handler.close() 

248 root.removeHandler(handler) 

249 

250 cls._fileHandlers.clear() 

251 cls._initialized = False 

252 cls.configState = [] 

253 

254 @classmethod 

255 def setLogLevels(cls, logLevels): 

256 """Set log level for one or more components or the root logger. 

257 

258 Parameters 

259 ---------- 

260 logLevels : `list` of `tuple` 

261 per-component logging levels, each item in the list is a tuple 

262 (component, level), `component` is a logger name or an empty string 

263 or `None` for default root logger, `level` is a logging level name, 

264 one of CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

265 

266 Notes 

267 ----- 

268 The special name ``.`` can be used to set the Python root 

269 logger. 

270 """ 

271 if isinstance(logLevels, dict): 

272 logLevels = logLevels.items() 

273 

274 # configure individual loggers 

275 for component, level in logLevels: 

276 cls._setLogLevel(component, level) 

277 # remember this call 

278 cls.configState.append((cls._setLogLevel, component, level)) 

279 

280 @classmethod 

281 def _setLogLevel(cls, component, level): 

282 """Set the log level for the given component. Record the current log 

283 level of the component so that it can be restored when resetting this 

284 log. 

285 

286 Parameters 

287 ---------- 

288 component : `str` or None 

289 The name of the log component or None for the default logger. 

290 The root logger can be specified either by an empty string or 

291 with the special name ``.``. 

292 level : `str` 

293 A valid python logging level. 

294 """ 

295 components: Set[Optional[str]] 

296 if component is None: 

297 components = cls.root_loggers() 

298 elif not component or component == ".": 

299 components = {None} 

300 else: 

301 components = {component} 

302 for component in components: 

303 cls._recordComponentSetting(component) 

304 if lsstLog is not None: 

305 lsstLogger = lsstLog.Log.getLogger(component or "") 

306 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

307 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level)) 

308 

309 @staticmethod 

310 def _getPyLogLevel(level): 

311 """Get the numeric value for the given log level name. 

312 

313 Parameters 

314 ---------- 

315 level : `str` 

316 One of the python `logging` log level names. 

317 

318 Returns 

319 ------- 

320 numericValue : `int` 

321 The python `logging` numeric value for the log level. 

322 """ 

323 if level == "VERBOSE": 

324 return VERBOSE 

325 elif level == "TRACE": 

326 return TRACE 

327 return getattr(logging, level, None) 

328 

329 @staticmethod 

330 def _getLsstLogLevel(level): 

331 """Get the numeric value for the given log level name. 

332 

333 If `lsst.log` is not setup this function will return `None` regardless 

334 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

335 will not be setup when `daf_butler` is setup. Packages that depend on 

336 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

337 

338 Parameters 

339 ---------- 

340 level : `str` 

341 One of the python `logging` log level names. 

342 

343 Returns 

344 ------- 

345 numericValue : `int` or `None` 

346 The `lsst.log` numeric value. 

347 

348 Notes 

349 ----- 

350 ``VERBOSE`` and ``TRACE`` logging are not supported by the LSST logger. 

351 ``VERBOSE`` will be converted to ``INFO`` and ``TRACE`` will be 

352 converted to ``DEBUG``. 

353 """ 

354 if lsstLog is None: 

355 return None 

356 if level == "VERBOSE": 

357 level = "INFO" 

358 elif level == "TRACE": 

359 level = "DEBUG" 

360 pylog_level = CliLog._getPyLogLevel(level) 

361 return lsstLog.LevelTranslator.logging2lsstLog(pylog_level) 

362 

363 class ComponentSettings: 

364 """Container for log level values for a logging component.""" 

365 

366 def __init__(self, component): 

367 self.component = component 

368 self.pythonLogLevel = logging.getLogger(component).level 

369 self.lsstLogLevel = ( 

370 lsstLog.Log.getLogger(component or "").getLevel() if lsstLog is not None else None 

371 ) 

372 if self.lsstLogLevel == -1: 

373 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

374 

375 def __repr__(self): 

376 return ( 

377 f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

378 f"lsstLogLevel={self.lsstLogLevel})" 

379 ) 

380 

381 @classmethod 

382 def _recordComponentSetting(cls, component): 

383 """Cache current levels for the given component in the list of 

384 component levels.""" 

385 componentSettings = cls.ComponentSettings(component) 

386 cls._componentSettings.append(componentSettings) 

387 

388 @classmethod 

389 def replayConfigState(cls, configState): 

390 """Re-create configuration using configuration state recorded earlier. 

391 

392 Parameters 

393 ---------- 

394 configState : `list` of `tuple` 

395 Tuples contain a method as first item and arguments for the method, 

396 in the same format as ``cls.configState``. 

397 """ 

398 if cls._initialized or cls.configState: 

399 # Already initialized, do not touch anything. 

400 log = logging.getLogger(__name__) 

401 log.warning("Log is already initialized, will not replay configuration.") 

402 return 

403 

404 # execute each one in order 

405 for call in configState: 

406 method, *args = call 

407 method(*args)