Coverage for python/lsst/utils/logging.py: 46%

111 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-26 06:05 -0700

1# This file is part of utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ( 

15 "TRACE", 

16 "VERBOSE", 

17 "getLogger", 

18 "getTraceLogger", 

19 "LsstLogAdapter", 

20 "PeriodicLogger", 

21 "trace_set_at", 

22) 

23 

24import logging 

25import sys 

26import time 

27from contextlib import contextmanager 

28from logging import LoggerAdapter 

29from typing import Any, Generator, List, Optional, Union 

30 

31try: 

32 import lsst.log.utils as logUtils 

33except ImportError: 

34 logUtils = None 

35 

36 

37# log level for trace (verbose debug). 

38TRACE = 5 

39logging.addLevelName(TRACE, "TRACE") 

40 

41# Verbose logging is midway between INFO and DEBUG. 

42VERBOSE = (logging.INFO + logging.DEBUG) // 2 

43logging.addLevelName(VERBOSE, "VERBOSE") 

44 

45 

46def _calculate_base_stacklevel(default: int, offset: int) -> int: 

47 """Calculate the default logging stacklevel to use. 

48 

49 Parameters 

50 ---------- 

51 default : `int` 

52 The stacklevel to use in Python 3.11 and newer where the only 

53 thing to take into account is the number of levels above the core 

54 Python logging infrastructure. 

55 offset : `int` 

56 The offset to apply for older Python implementations that need to 

57 take into account internal call stacks. 

58 

59 Returns 

60 ------- 

61 stacklevel : `int` 

62 The stack level to pass to internal logging APIs that should result 

63 in the log messages being reported in caller lines. 

64 

65 Notes 

66 ----- 

67 In Python 3.11 the logging infrastructure was fixed such that we no 

68 longer need to understand that a LoggerAdapter log messages need to 

69 have a stack level one higher than a Logger would need. ``stacklevel=1`` 

70 now always means "log from the caller's line" without the caller having 

71 to understand internal implementation details. 

72 """ 

73 stacklevel = default 

74 if sys.version_info < (3, 11, 0): 74 ↛ 76line 74 didn't jump to line 76, because the condition on line 74 was never false

75 stacklevel += offset 

76 return stacklevel 

77 

78 

79def trace_set_at(name: str, number: int) -> None: 

80 """Adjust logging level to display messages with the trace number being 

81 less than or equal to the provided value. 

82 

83 Parameters 

84 ---------- 

85 name : `str` 

86 Name of the logger. 

87 number : `int` 

88 The trace number threshold for display. 

89 

90 Examples 

91 -------- 

92 .. code-block:: python 

93 

94 lsst.utils.logging.trace_set_at("lsst.afw", 3) 

95 

96 This will set loggers ``TRACE0.lsst.afw`` to ``TRACE3.lsst.afw`` to 

97 ``DEBUG`` and ``TRACE4.lsst.afw`` and ``TRACE5.lsst.afw`` to ``INFO``. 

98 

99 Notes 

100 ----- 

101 Loggers ``TRACE0.`` to ``TRACE5.`` are set. All loggers above 

102 the specified threshold are set to ``INFO`` and those below the threshold 

103 are set to ``DEBUG``. The expectation is that ``TRACE`` loggers only 

104 issue ``DEBUG`` log messages. 

105 

106 If ``lsst.log`` is installed, this function will also call 

107 `lsst.log.utils.traceSetAt` to ensure that non-Python loggers are 

108 also configured correctly. 

109 """ 

110 for i in range(6): 

111 level = logging.INFO if i > number else logging.DEBUG 

112 getTraceLogger(name, i).setLevel(level) 

113 

114 # if lsst log is available also set the trace loggers there. 

115 if logUtils is not None: 

116 logUtils.traceSetAt(name, number) 

117 

118 

119class _F: 

120 """Format, supporting `str.format()` syntax. 

121 

122 Notes 

123 ----- 

124 This follows the recommendation from 

125 https://docs.python.org/3/howto/logging-cookbook.html#using-custom-message-objects 

126 """ 

127 

128 def __init__(self, fmt: str, /, *args: Any, **kwargs: Any): 

129 self.fmt = fmt 

130 self.args = args 

131 self.kwargs = kwargs 

132 

133 def __str__(self) -> str: 

134 return self.fmt.format(*self.args, **self.kwargs) 

135 

136 

137class LsstLogAdapter(LoggerAdapter): 

138 """A special logging adapter to provide log features for LSST code. 

139 

140 Expected to be instantiated initially by a call to `getLogger()`. 

141 

142 This class provides enhancements over `logging.Logger` that include: 

143 

144 * Methods for issuing trace and verbose level log messages. 

145 * Provision of a context manager to temporarily change the log level. 

146 * Attachment of logging level constants to the class to make it easier 

147 for a Task writer to access a specific log level without having to 

148 know the underlying logger class. 

149 """ 

150 

151 # Store logging constants in the class for convenience. This is not 

152 # something supported by Python loggers but can simplify some 

153 # logic if the logger is available. 

154 CRITICAL = logging.CRITICAL 

155 ERROR = logging.ERROR 

156 DEBUG = logging.DEBUG 

157 INFO = logging.INFO 

158 WARNING = logging.WARNING 

159 

160 # Python supports these but prefers they are not used. 

161 FATAL = logging.FATAL 

162 WARN = logging.WARN 

163 

164 # These are specific to Tasks 

165 TRACE = TRACE 

166 VERBOSE = VERBOSE 

167 

168 # The stack level to use when issuing log messages. For Python 3.11 

169 # this is generally 2 (this method and the internal infrastructure). 

170 # For older python we need one higher because of the extra indirection 

171 # via LoggingAdapter internals. 

172 _stacklevel = _calculate_base_stacklevel(2, 1) 

173 

174 @contextmanager 

175 def temporary_log_level(self, level: Union[int, str]) -> Generator: 

176 """Temporarily set the level of this logger. 

177 

178 Parameters 

179 ---------- 

180 level : `int` 

181 The new temporary log level. 

182 """ 

183 old = self.level 

184 self.setLevel(level) 

185 try: 

186 yield 

187 finally: 

188 self.setLevel(old) 

189 

190 @property 

191 def level(self) -> int: 

192 """Return current level of this logger (``int``).""" 

193 return self.logger.level 

194 

195 def getChild(self, name: str) -> LsstLogAdapter: 

196 """Get the named child logger. 

197 

198 Parameters 

199 ---------- 

200 name : `str` 

201 Name of the child relative to this logger. 

202 

203 Returns 

204 ------- 

205 child : `LsstLogAdapter` 

206 The child logger. 

207 """ 

208 return getLogger(name=name, logger=self.logger) 

209 

210 def _process_stacklevel(self, kwargs: dict[str, Any], offset: int = 0) -> int: 

211 # Return default stacklevel, taking into account kwargs[stacklevel]. 

212 stacklevel = self._stacklevel 

213 if "stacklevel" in kwargs: 

214 # External user expects stacklevel=1 to mean "report from their 

215 # line" but the code here is already trying to achieve that by 

216 # default. Therefore if an external stacklevel is specified we 

217 # adjust their stacklevel request by 1. 

218 stacklevel = stacklevel + kwargs.pop("stacklevel") - 1 

219 

220 # The offset can be used to indicate that we have to take into account 

221 # additional internal layers before calling python logging. 

222 return _calculate_base_stacklevel(stacklevel, offset) 

223 

224 def fatal(self, msg: str, *args: Any, **kwargs: Any) -> None: 

225 # Python does not provide this method in LoggerAdapter but does 

226 # not formally deprecate it in favor of critical() either. 

227 # Provide it without deprecation message for consistency with Python. 

228 # Have to adjust stacklevel on Python 3.10 and older to account 

229 # for call through self.critical. 

230 stacklevel = self._process_stacklevel(kwargs, offset=1) 

231 self.critical(msg, *args, **kwargs, stacklevel=stacklevel) 

232 

233 def verbose(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

234 """Issue a VERBOSE level log message. 

235 

236 Arguments are as for `logging.info`. 

237 ``VERBOSE`` is between ``DEBUG`` and ``INFO``. 

238 """ 

239 # There is no other way to achieve this other than a special logger 

240 # method. 

241 # Stacklevel is passed in so that the correct line is reported 

242 stacklevel = self._process_stacklevel(kwargs) 

243 self.log(VERBOSE, fmt, *args, **kwargs, stacklevel=stacklevel) 

244 

245 def trace(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

246 """Issue a TRACE level log message. 

247 

248 Arguments are as for `logging.info`. 

249 ``TRACE`` is lower than ``DEBUG``. 

250 """ 

251 # There is no other way to achieve this other than a special logger 

252 # method. 

253 stacklevel = self._process_stacklevel(kwargs) 

254 self.log(TRACE, fmt, *args, **kwargs, stacklevel=stacklevel) 

255 

256 def setLevel(self, level: Union[int, str]) -> None: 

257 """Set the level for the logger, trapping lsst.log values. 

258 

259 Parameters 

260 ---------- 

261 level : `int` 

262 The level to use. If the level looks too big to be a Python 

263 logging level it is assumed to be a lsst.log level. 

264 """ 

265 if isinstance(level, int) and level > logging.CRITICAL: 

266 self.logger.warning( 

267 "Attempting to set level to %d -- looks like an lsst.log level so scaling it accordingly.", 

268 level, 

269 ) 

270 level //= 1000 

271 

272 self.logger.setLevel(level) 

273 

274 @property 

275 def handlers(self) -> List[logging.Handler]: 

276 """Log handlers associated with this logger.""" 

277 return self.logger.handlers 

278 

279 def addHandler(self, handler: logging.Handler) -> None: 

280 """Add a handler to this logger. 

281 

282 The handler is forwarded to the underlying logger. 

283 """ 

284 self.logger.addHandler(handler) 

285 

286 def removeHandler(self, handler: logging.Handler) -> None: 

287 """Remove the given handler from the underlying logger.""" 

288 self.logger.removeHandler(handler) 

289 

290 

291def getLogger(name: Optional[str] = None, logger: Optional[logging.Logger] = None) -> LsstLogAdapter: 

292 """Get a logger compatible with LSST usage. 

293 

294 Parameters 

295 ---------- 

296 name : `str`, optional 

297 Name of the logger. Root logger if `None`. 

298 logger : `logging.Logger` or `LsstLogAdapter` 

299 If given the logger is converted to the relevant logger class. 

300 If ``name`` is given the logger is assumed to be a child of the 

301 supplied logger. 

302 

303 Returns 

304 ------- 

305 logger : `LsstLogAdapter` 

306 The relevant logger. 

307 

308 Notes 

309 ----- 

310 A `logging.LoggerAdapter` is used since it is easier to provide a more 

311 uniform interface than when using `logging.setLoggerClass`. An adapter 

312 can be wrapped around the root logger and the `~logging.setLoggerClass` 

313 will return the logger first given that name even if the name was 

314 used before the `Task` was created. 

315 """ 

316 if not logger: 

317 logger = logging.getLogger(name) 

318 elif name: 

319 logger = logger.getChild(name) 

320 return LsstLogAdapter(logger, {}) 

321 

322 

323LsstLoggers = Union[logging.Logger, LsstLogAdapter] 

324 

325 

326def getTraceLogger(logger: Union[str, LsstLoggers], trace_level: int) -> LsstLogAdapter: 

327 """Get a logger with the appropriate TRACE name. 

328 

329 Parameters 

330 ---------- 

331 logger : `logging.Logger` or `LsstLogAdapter` or `lsst.log.Log` or `str` 

332 A logger to be used to derive the new trace logger. Can be a logger 

333 object that supports the ``name`` property or a string. 

334 trace_level : `int` 

335 The trace level to use for the logger. 

336 

337 Returns 

338 ------- 

339 trace_logger : `LsstLogAdapter` 

340 A new trace logger. The name will be derived by pre-pending ``TRACEn.`` 

341 to the name of the supplied logger. If the root logger is given 

342 the returned logger will be named ``TRACEn``. 

343 """ 

344 name = getattr(logger, "name", str(logger)) 

345 trace_name = f"TRACE{trace_level}.{name}" if name else f"TRACE{trace_level}" 

346 return getLogger(trace_name) 

347 

348 

349class PeriodicLogger: 

350 """Issue log messages if a time threshold has elapsed. 

351 

352 This class can be used in long-running sections of code where it would 

353 be useful to issue a log message periodically to show that the 

354 algorithm is progressing. 

355 

356 Parameters 

357 ---------- 

358 logger : `logging.Logger` or `LsstLogAdapter` 

359 Logger to use when issuing a message. 

360 interval : `float` 

361 The minimum interval between log messages. If `None` the class 

362 default will be used. 

363 level : `int`, optional 

364 Log level to use when issuing messages. 

365 """ 

366 

367 LOGGING_INTERVAL = 600.0 

368 """Default interval between log messages.""" 

369 

370 def __init__(self, logger: LsstLoggers, interval: Optional[float] = None, level: int = VERBOSE): 

371 self.logger = logger 

372 self.interval = interval if interval is not None else self.LOGGING_INTERVAL 

373 self.level = level 

374 self.next_log_time = time.time() + self.interval 

375 self.num_issued = 0 

376 

377 # The stacklevel we need to issue logs is determined by the type 

378 # of logger we have been given. A LoggerAdapter has an extra 

379 # level of indirection. In Python 3.11 the logging infrastructure 

380 # takes care to check for internal logging stack frames so there 

381 # is no need for a difference. 

382 self._stacklevel = _calculate_base_stacklevel(2, 1 if isinstance(self.logger, LoggerAdapter) else 0) 

383 

384 def log(self, msg: str, *args: Any) -> bool: 

385 """Issue a log message if the interval has elapsed. 

386 

387 Parameters 

388 ---------- 

389 msg : `str` 

390 Message to issue if the time has been exceeded. 

391 *args : Any 

392 Parameters to be passed to the log system. 

393 

394 Returns 

395 ------- 

396 issued : `bool` 

397 Returns `True` if a log message was sent to the logging system. 

398 Returns `False` if the interval has not yet elapsed. Returning 

399 `True` does not indicate whether the log message was in fact 

400 issued by the logging system. 

401 """ 

402 if (current_time := time.time()) > self.next_log_time: 

403 self.logger.log(self.level, msg, *args, stacklevel=self._stacklevel) 

404 self.next_log_time = current_time + self.interval 

405 self.num_issued += 1 

406 return True 

407 return False