Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import datetime 

23import logging 

24 

25try: 

26 import lsst.log as lsstLog 

27except ModuleNotFoundError: 

28 lsstLog = None 

29 

30from lsst.daf.butler import ButlerMDC 

31from ..core.logging import MDCDict 

32 

33 

34class PrecisionLogFormatter(logging.Formatter): 

35 """A log formatter that issues accurate timezone-aware timestamps.""" 

36 

37 converter = datetime.datetime.fromtimestamp 

38 

39 use_local = True 

40 """Control whether local time is displayed instead of UTC.""" 

41 

42 def formatTime(self, record, datefmt=None): 

43 """Format the time as an aware datetime.""" 

44 ct = self.converter(record.created, tz=datetime.timezone.utc) 

45 if self.use_local: 

46 ct = ct.astimezone() 

47 if datefmt: 

48 s = ct.strftime(datefmt) 

49 else: 

50 s = ct.isoformat(sep='T', timespec='milliseconds') 

51 return s 

52 

53 

54class CliLog: 

55 """Interface for managing python logging and ``lsst.log``. 

56 

57 This class defines log format strings for the log output and timestamp 

58 formats. It also configures ``lsst.log`` to forward all messages to 

59 Python `logging`. 

60 

61 This class can perform log uninitialization, which allows command line 

62 interface code that initializes logging to run unit tests that execute in 

63 batches, without affecting other unit tests. See ``resetLog``.""" 

64 

65 defaultLsstLogLevel = lsstLog.FATAL if lsstLog is not None else None 

66 

67 pylog_longLogFmt = "{levelname} {asctime} {name} ({MDC[LABEL]})({filename}:{lineno}) - {message}" 

68 """The log format used when the lsst.log package is not importable and the 

69 log is initialized with longlog=True.""" 

70 

71 pylog_normalFmt = "{name} {levelname}: {message}" 

72 """The log format used when the lsst.log package is not importable and the 

73 log is initialized with longlog=False.""" 

74 

75 configState = [] 

76 """Configuration state. Contains tuples where first item in a tuple is 

77 a method and remaining items are arguments for the method. 

78 """ 

79 

80 _initialized = False 

81 _componentSettings = [] 

82 

83 @classmethod 

84 def initLog(cls, longlog): 

85 """Initialize logging. This should only be called once per program 

86 execution. After the first call this will log a warning and return. 

87 

88 If lsst.log is importable, will add its log handler to the python 

89 root logger's handlers. 

90 

91 Parameters 

92 ---------- 

93 longlog : `bool` 

94 If True, make log messages appear in long format, by default False. 

95 """ 

96 if cls._initialized: 

97 # Unit tests that execute more than one command do end up 

98 # calling this function multiple times in one program execution, 

99 # so do log a debug but don't log an error or fail, just make the 

100 # re-initialization a no-op. 

101 log = logging.getLogger(__name__) 

102 log.debug("Log is already initialized, returning without re-initializing.") 

103 return 

104 cls._initialized = True 

105 cls._recordComponentSetting(None) 

106 

107 if lsstLog is not None: 

108 # Ensure that log messages are forwarded back to python. 

109 # Disable use of lsst.log MDC -- we expect butler uses to 

110 # use ButlerMDC. 

111 lsstLog.configure_pylog_MDC("DEBUG", MDC_class=None) 

112 

113 # Forward python lsst.log messages directly to python logging. 

114 # This can bypass the C++ layer entirely but requires that 

115 # MDC is set via ButlerMDC, rather than in lsst.log. 

116 lsstLog.usePythonLogging() 

117 

118 if longlog: 

119 

120 # Want to create our own Formatter so that we can get high 

121 # precision timestamps. This requires we attach our own 

122 # default stream handler. 

123 defaultHandler = logging.StreamHandler() 

124 formatter = PrecisionLogFormatter(fmt=cls.pylog_longLogFmt, style="{") 

125 defaultHandler.setFormatter(formatter) 

126 

127 logging.basicConfig(level=logging.INFO, 

128 force=True, 

129 handlers=[defaultHandler], 

130 ) 

131 

132 else: 

133 logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt, style="{") 

134 

135 # Initialize root logger level. 

136 cls._setLogLevel(None, "INFO") 

137 

138 # also capture warnings and send them to logging 

139 logging.captureWarnings(True) 

140 

141 # Create a record factory that ensures that an MDC is attached 

142 # to the records. By default this is only used for long-log 

143 # but always enable it for when someone adds a new handler 

144 # that needs it. 

145 old_factory = logging.getLogRecordFactory() 

146 

147 def record_factory(*args, **kwargs): 

148 record = old_factory(*args, **kwargs) 

149 # Make sure we send a copy of the global dict in the record. 

150 record.MDC = MDCDict(ButlerMDC._MDC) 

151 return record 

152 

153 logging.setLogRecordFactory(record_factory) 

154 

155 # remember this call 

156 cls.configState.append((cls.initLog, longlog)) 

157 

158 @classmethod 

159 def resetLog(cls): 

160 """Uninitialize the butler CLI Log handler and reset component log 

161 levels. 

162 

163 If the lsst.log handler was added to the python root logger's handlers 

164 in `initLog`, it will be removed here. 

165 

166 For each logger level that was set by this class, sets that logger's 

167 level to the value it was before this class set it. For lsst.log, if a 

168 component level was uninitialized, it will be set to 

169 `Log.defaultLsstLogLevel` because there is no log4cxx api to set a 

170 component back to an uninitialized state. 

171 """ 

172 if lsstLog: 

173 lsstLog.doNotUsePythonLogging() 

174 for componentSetting in reversed(cls._componentSettings): 

175 if lsstLog is not None and componentSetting.lsstLogLevel is not None: 

176 lsstLog.setLevel(componentSetting.component or "", componentSetting.lsstLogLevel) 

177 logger = logging.getLogger(componentSetting.component) 

178 logger.setLevel(componentSetting.pythonLogLevel) 

179 cls._setLogLevel(None, "INFO") 

180 cls._initialized = False 

181 cls.configState = [] 

182 

183 @classmethod 

184 def setLogLevels(cls, logLevels): 

185 """Set log level for one or more components or the root logger. 

186 

187 Parameters 

188 ---------- 

189 logLevels : `list` of `tuple` 

190 per-component logging levels, each item in the list is a tuple 

191 (component, level), `component` is a logger name or an empty string 

192 or `None` for root logger, `level` is a logging level name, one of 

193 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive). 

194 """ 

195 if isinstance(logLevels, dict): 

196 logLevels = logLevels.items() 

197 

198 # configure individual loggers 

199 for component, level in logLevels: 

200 cls._setLogLevel(component, level) 

201 # remember this call 

202 cls.configState.append((cls._setLogLevel, component, level)) 

203 

204 @classmethod 

205 def _setLogLevel(cls, component, level): 

206 """Set the log level for the given component. Record the current log 

207 level of the component so that it can be restored when resetting this 

208 log. 

209 

210 Parameters 

211 ---------- 

212 component : `str` or None 

213 The name of the log component or None for the root logger. 

214 level : `str` 

215 A valid python logging level. 

216 """ 

217 cls._recordComponentSetting(component) 

218 if lsstLog is not None: 

219 lsstLogger = lsstLog.Log.getLogger(component or "") 

220 lsstLogger.setLevel(cls._getLsstLogLevel(level)) 

221 logging.getLogger(component or None).setLevel(cls._getPyLogLevel(level)) 

222 

223 @staticmethod 

224 def _getPyLogLevel(level): 

225 """Get the numeric value for the given log level name. 

226 

227 Parameters 

228 ---------- 

229 level : `str` 

230 One of the python `logging` log level names. 

231 

232 Returns 

233 ------- 

234 numericValue : `int` 

235 The python `logging` numeric value for the log level. 

236 """ 

237 if level == "VERBOSE": 

238 from .. import VERBOSE 

239 return VERBOSE 

240 return getattr(logging, level, None) 

241 

242 @staticmethod 

243 def _getLsstLogLevel(level): 

244 """Get the numeric value for the given log level name. 

245 

246 If `lsst.log` is not setup this function will return `None` regardless 

247 of input. `daf_butler` does not directly depend on `lsst.log` and so it 

248 will not be setup when `daf_butler` is setup. Packages that depend on 

249 `daf_butler` and use `lsst.log` may setup `lsst.log`. 

250 

251 Will adapt the python name to an `lsst.log` name: 

252 - CRITICAL to FATAL 

253 - WARNING to WARN 

254 

255 Parameters 

256 ---------- 

257 level : `str` 

258 One of the python `logging` log level names. 

259 

260 Returns 

261 ------- 

262 numericValue : `int` or `None` 

263 The `lsst.log` numeric value. 

264 """ 

265 if lsstLog is None: 

266 return None 

267 if level == "CRITICAL": 

268 level = "FATAL" 

269 elif level == "WARNING": 

270 level = "WARN" 

271 elif level == "VERBOSE": 

272 # LSST log does not yet have verbose defined 

273 return (lsstLog.Log.DEBUG + lsstLog.Log.INFO) // 2 

274 return getattr(lsstLog.Log, level, None) 

275 

276 class ComponentSettings: 

277 """Container for log level values for a logging component.""" 

278 def __init__(self, component): 

279 self.component = component 

280 self.pythonLogLevel = logging.getLogger(component).level 

281 self.lsstLogLevel = (lsstLog.Log.getLogger(component or "").getLevel() 

282 if lsstLog is not None else None) 

283 if self.lsstLogLevel == -1: 

284 self.lsstLogLevel = CliLog.defaultLsstLogLevel 

285 

286 def __repr__(self): 

287 return (f"ComponentSettings(component={self.component}, pythonLogLevel={self.pythonLogLevel}, " 

288 f"lsstLogLevel={self.lsstLogLevel})") 

289 

290 @classmethod 

291 def _recordComponentSetting(cls, component): 

292 """Cache current levels for the given component in the list of 

293 component levels.""" 

294 componentSettings = cls.ComponentSettings(component) 

295 cls._componentSettings.append(componentSettings) 

296 

297 @classmethod 

298 def replayConfigState(cls, configState): 

299 """Re-create configuration using configuration state recorded earlier. 

300 

301 Parameters 

302 ---------- 

303 configState : `list` of `tuple` 

304 Tuples contain a method as first item and arguments for the method, 

305 in the same format as ``cls.configState``. 

306 """ 

307 if cls._initialized or cls.configState: 

308 # Already initialized, do not touch anything. 

309 log = logging.getLogger(__name__) 

310 log.warning("Log is already initialized, will not replay configuration.") 

311 return 

312 

313 # execute each one in order 

314 for call in configState: 

315 method, *args = call 

316 method(*args)