Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import datetime 

23import logging 

24 

25try: 

26 import lsst.log as lsstLog 

27except ModuleNotFoundError: 

28 lsstLog = None 

29 

30from ..core.logging import JsonLogFormatter, ButlerMDC 

31 

32 

33class PrecisionLogFormatter(logging.Formatter): 

34 """A log formatter that issues accurate timezone-aware timestamps.""" 

35 

36 converter = datetime.datetime.fromtimestamp 

37 

38 use_local = True 

39 """Control whether local time is displayed instead of UTC.""" 

40 

41 def formatTime(self, record, datefmt=None): 

42 """Format the time as an aware datetime.""" 

43 ct = self.converter(record.created, tz=datetime.timezone.utc) 

44 if self.use_local: 

45 ct = ct.astimezone() 

46 if datefmt: 

47 s = ct.strftime(datefmt) 

48 else: 

49 s = ct.isoformat(sep='T', timespec='milliseconds') 

50 return s 

51 

52 

53class CliLog: 

54 """Interface for managing python logging and ``lsst.log``. 

55 

56 This class defines log format strings for the log output and timestamp 

57 formats. It also configures ``lsst.log`` to forward all messages to 

58 Python `logging`. 

59 

60 This class can perform log uninitialization, which allows command line 

61 interface code that initializes logging to run unit tests that execute in 

62 batches, without affecting other unit tests. See ``resetLog``.""" 

63 

64 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

65 

66 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

67 """The log format used when the lsst.log package is not importable and the 

68 log is initialized with longlog=True.""" 

69 

70 pylog_normalFmt = "{name} {levelname}: {message}" 

71 """The log format used when the lsst.log package is not importable and the 

72 log is initialized with longlog=False.""" 

73 

74 configState = [] 

75 """Configuration state. Contains tuples where first item in a tuple is 

76 a method and remaining items are arguments for the method. 

77 """ 

78 

79 _initialized = False 

80 _componentSettings = [] 

81 

82 _fileHandlers = [] 

83 """Any FileHandler classes attached to the root logger by this class 

84 that need to be closed on reset.""" 

85 

86 @classmethod 

87 def initLog(cls, longlog: bool, log_tty: bool = True, log_file=()): 

88 """Initialize logging. This should only be called once per program 

89 execution. After the first call this will log a warning and return. 

90 

91 If lsst.log is importable, will add its log handler to the python 

92 root logger's handlers. 

93 

94 Parameters 

95 ---------- 

96 longlog : `bool` 

97 If True, make log messages appear in long format, by default False. 

98 log_tty : `bool` 

99 Control whether a default stream handler is enabled that logs 

100 to the terminal. 

101 log_file : `tuple` of `str` 

102 Path to files to write log records. If path ends in ``.json`` the 

103 records will be written in JSON format. Else they will be written 

104 in text format. If empty no log file will be created. Records 

105 will be appended to this file if it exists. 

106 """ 

107 if cls._initialized: 

108 # Unit tests that execute more than one command do end up 

109 # calling this function multiple times in one program execution, 

110 # so do log a debug but don't log an error or fail, just make the 

111 # re-initialization a no-op. 

112 log = logging.getLogger(__name__) 

113 log.debug("Log is already initialized, returning without re-initializing.") 

114 return 

115 cls._initialized = True 

116 cls._recordComponentSetting(None) 

117 

118 if lsstLog is not None: 

119 # Ensure that log messages are forwarded back to python. 

120 # Disable use of lsst.log MDC -- we expect butler uses to 

121 # use ButlerMDC. 

122 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

123 

124 # Forward python lsst.log messages directly to python logging. 

125 # This can bypass the C++ layer entirely but requires that 

126 # MDC is set via ButlerMDC, rather than in lsst.log. 

127 lsstLog.usePythonLogging() 

128 

129 if not log_tty: 

130 logging.basicConfig(force=True, handlers=[logging.NullHandler()]) 

131 elif longlog: 

132 

133 # Want to create our own Formatter so that we can get high 

134 # precision timestamps. This requires we attach our own 

135 # default stream handler. 

136 defaultHandler = logging.StreamHandler() 

137 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

138 defaultHandler.setFormatter(formatter) 

139 

140 logging.basicConfig(level=logging.INFO, 

141 force=True, 

142 handlers=[defaultHandler], 

143 ) 

144 

145 else: 

146 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{") 

147 

148 # Initialize root logger level. 

149 cls._setLogLevel(None, "INFO") 

150 

151 # also capture warnings and send them to logging 

152 logging.captureWarnings(True) 

153 

154 # Create a record factory that ensures that an MDC is attached 

155 # to the records. By default this is only used for long-log 

156 # but always enable it for when someone adds a new handler 

157 # that needs it. 

158 ButlerMDC.add_mdc_log_record_factory() 

159 

160 # Set up the file logger 

161 for file in log_file: 

162 handler = logging.FileHandler(file) 

163 if file.endswith(".json"): 

164 formatter = JsonLogFormatter() 

165 else: 

166 if longlog: 

167 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

168 else: 

169 formatter = logging.Formatter(fmt=cls.pylog_normalFmt, style="{") 

170 handler.setFormatter(formatter) 

171 logging.getLogger().addHandler(handler) 

172 cls._fileHandlers.append(handler) 

173 

174 # remember this call 

175 cls.configState.append((cls.initLog, longlog, log_tty, log_file)) 

176 

177 @classmethod 

178 def resetLog(cls): 

179 """Uninitialize the butler CLI Log handler and reset component log 

180 levels. 

181 

182 If the lsst.log handler was added to the python root logger's handlers 

183 in `initLog`, it will be removed here. 

184 

185 For each logger level that was set by this class, sets that logger's 

186 level to the value it was before this class set it. For lsst.log, if a 

187 component level was uninitialized, it will be set to 

188 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

189 component back to an uninitialized state. 

190 """ 

191 if lsstLog: 

192 lsstLog.doNotUsePythonLogging() 

193 for componentSetting in reversed(cls._componentSettings): 

194 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

195 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

196 logger = logging.getLogger(componentSetting.component) 

197 logger.setLevel(componentSetting.pythonLogLevel) 

198 cls._setLogLevel(None, "INFO") 

199 

200 ButlerMDC.restore_log_record_factory() 

201 

202 # Remove the FileHandler we may have attached. 

203 root = logging.getLogger() 

204 for handler in cls._fileHandlers: 

205 handler.close() 

206 root.removeHandler(handler) 

207 

208 cls._fileHandlers.clear() 

209 cls._initialized = False 

210 cls.configState = [] 

211 

212 @classmethod 

213 def setLogLevels(cls, logLevels): 

214 """Set log level for one or more components or the root logger. 

215 

216 Parameters 

217 ---------- 

218 logLevels : `list` of `tuple` 

219 per-component logging levels, each item in the list is a tuple 

220 (component, level), `component` is a logger name or an empty string 

221 or `None` for root logger, `level` is a logging level name, one of 

222 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

223 """ 

224 if isinstance(logLevels, dict): 

225 logLevels = logLevels.items() 

226 

227 # configure individual loggers 

228 for component, level in logLevels: 

229 cls._setLogLevel(component, level) 

230 # remember this call 

231 cls.configState.append((cls._setLogLevel, component, level)) 

232 

233 @classmethod 

234 def _setLogLevel(cls, component, level): 

235 """Set the log level for the given component. Record the current log 

236 level of the component so that it can be restored when resetting this 

237 log. 

238 

239 Parameters 

240 ---------- 

241 component : `str` or None 

242 The name of the log component or None for the root logger. 

243 level : `str` 

244 A valid python logging level. 

245 """ 

246 cls._recordComponentSetting(component) 

247 if lsstLog is not None: 

248 lsstLogger = lsstLog.Log.getLogger(component or "") 

249 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

250 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level)) 

251 

252 @staticmethod 

253 def _getPyLogLevel(level): 

254 """Get the numeric value for the given log level name. 

255 

256 Parameters 

257 ---------- 

258 level : `str` 

259 One of the python `logging` log level names. 

260 

261 Returns 

262 ------- 

263 numericValue : `int` 

264 The python `logging` numeric value for the log level. 

265 """ 

266 if level == "VERBOSE": 

267 from .. import VERBOSE 

268 return VERBOSE 

269 return getattr(logging, level, None) 

270 

271 @staticmethod 

272 def _getLsstLogLevel(level): 

273 """Get the numeric value for the given log level name. 

274 

275 If `lsst.log` is not setup this function will return `None` regardless 

276 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

277 will not be setup when `daf_butler` is setup. Packages that depend on 

278 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

279 

280 Will adapt the python name to an `lsst.log` name: 

281 - CRITICAL to FATAL 

282 - WARNING to WARN 

283 

284 Parameters 

285 ---------- 

286 level : `str` 

287 One of the python `logging` log level names. 

288 

289 Returns 

290 ------- 

291 numericValue : `int` or `None` 

292 The `lsst.log` numeric value. 

293 """ 

294 if lsstLog is None: 

295 return None 

296 if level == "CRITICAL": 

297 level = "FATAL" 

298 elif level == "WARNING": 

299 level = "WARN" 

300 elif level == "VERBOSE": 

301 # LSST log does not yet have verbose defined 

302 return (lsstLog.Log.DEBUG + lsstLog.Log.INFO) // 2 

303 return getattr(lsstLog.Log, level, None) 

304 

305 class ComponentSettings: 

306 """Container for log level values for a logging component.""" 

307 def __init__(self, component): 

308 self.component = component 

309 self.pythonLogLevel = logging.getLogger(component).level 

310 self.lsstLogLevel = (lsstLog.Log.getLogger(component or "").getLevel() 

311 if lsstLog is not None else None) 

312 if self.lsstLogLevel == -1: 

313 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

314 

315 def __repr__(self): 

316 return (f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

317 f"lsstLogLevel={self.lsstLogLevel})") 

318 

319 @classmethod 

320 def _recordComponentSetting(cls, component): 

321 """Cache current levels for the given component in the list of 

322 component levels.""" 

323 componentSettings = cls.ComponentSettings(component) 

324 cls._componentSettings.append(componentSettings) 

325 

326 @classmethod 

327 def replayConfigState(cls, configState): 

328 """Re-create configuration using configuration state recorded earlier. 

329 

330 Parameters 

331 ---------- 

332 configState : `list` of `tuple` 

333 Tuples contain a method as first item and arguments for the method, 

334 in the same format as ``cls.configState``. 

335 """ 

336 if cls._initialized or cls.configState: 

337 # Already initialized, do not touch anything. 

338 log = logging.getLogger(__name__) 

339 log.warning("Log is already initialized, will not replay configuration.") 

340 return 

341 

342 # execute each one in order 

343 for call in configState: 

344 method, *args = call 

345 method(*args)