Coverage for python/lsst/utils/logging.py: 47%

112 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-08 09:53 +0000

1# This file is part of utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ( 

15 "TRACE", 

16 "VERBOSE", 

17 "getLogger", 

18 "getTraceLogger", 

19 "LsstLogAdapter", 

20 "PeriodicLogger", 

21 "trace_set_at", 

22) 

23 

24import logging 

25import sys 

26import time 

27from collections.abc import Generator 

28from contextlib import contextmanager 

29from logging import LoggerAdapter 

30from typing import Any, Union 

31 

32try: 

33 import lsst.log.utils as logUtils 

34except ImportError: 

35 logUtils = None 

36 

37 

38# log level for trace (verbose debug). 

39TRACE = 5 

40logging.addLevelName(TRACE, "TRACE") 

41 

42# Verbose logging is midway between INFO and DEBUG. 

43VERBOSE = (logging.INFO + logging.DEBUG) // 2 

44logging.addLevelName(VERBOSE, "VERBOSE") 

45 

46 

47def _calculate_base_stacklevel(default: int, offset: int) -> int: 

48 """Calculate the default logging stacklevel to use. 

49 

50 Parameters 

51 ---------- 

52 default : `int` 

53 The stacklevel to use in Python 3.11 and newer where the only 

54 thing to take into account is the number of levels above the core 

55 Python logging infrastructure. 

56 offset : `int` 

57 The offset to apply for older Python implementations that need to 

58 take into account internal call stacks. 

59 

60 Returns 

61 ------- 

62 stacklevel : `int` 

63 The stack level to pass to internal logging APIs that should result 

64 in the log messages being reported in caller lines. 

65 

66 Notes 

67 ----- 

68 In Python 3.11 the logging infrastructure was fixed such that we no 

69 longer need to understand that a LoggerAdapter log messages need to 

70 have a stack level one higher than a Logger would need. ``stacklevel=1`` 

71 now always means "log from the caller's line" without the caller having 

72 to understand internal implementation details. 

73 """ 

74 stacklevel = default 

75 if sys.version_info < (3, 11, 0): 75 ↛ 77line 75 didn't jump to line 77, because the condition on line 75 was never false

76 stacklevel += offset 

77 return stacklevel 

78 

79 

80def trace_set_at(name: str, number: int) -> None: 

81 """Adjust logging level to display messages with the trace number being 

82 less than or equal to the provided value. 

83 

84 Parameters 

85 ---------- 

86 name : `str` 

87 Name of the logger. 

88 number : `int` 

89 The trace number threshold for display. 

90 

91 Examples 

92 -------- 

93 .. code-block:: python 

94 

95 lsst.utils.logging.trace_set_at("lsst.afw", 3) 

96 

97 This will set loggers ``TRACE0.lsst.afw`` to ``TRACE3.lsst.afw`` to 

98 ``DEBUG`` and ``TRACE4.lsst.afw`` and ``TRACE5.lsst.afw`` to ``INFO``. 

99 

100 Notes 

101 ----- 

102 Loggers ``TRACE0.`` to ``TRACE5.`` are set. All loggers above 

103 the specified threshold are set to ``INFO`` and those below the threshold 

104 are set to ``DEBUG``. The expectation is that ``TRACE`` loggers only 

105 issue ``DEBUG`` log messages. 

106 

107 If ``lsst.log`` is installed, this function will also call 

108 `lsst.log.utils.traceSetAt` to ensure that non-Python loggers are 

109 also configured correctly. 

110 """ 

111 for i in range(6): 

112 level = logging.INFO if i > number else logging.DEBUG 

113 getTraceLogger(name, i).setLevel(level) 

114 

115 # if lsst log is available also set the trace loggers there. 

116 if logUtils is not None: 

117 logUtils.traceSetAt(name, number) 

118 

119 

120class _F: 

121 """Format, supporting `str.format()` syntax. 

122 

123 Notes 

124 ----- 

125 This follows the recommendation from 

126 https://docs.python.org/3/howto/logging-cookbook.html#using-custom-message-objects 

127 """ 

128 

129 def __init__(self, fmt: str, /, *args: Any, **kwargs: Any): 

130 self.fmt = fmt 

131 self.args = args 

132 self.kwargs = kwargs 

133 

134 def __str__(self) -> str: 

135 return self.fmt.format(*self.args, **self.kwargs) 

136 

137 

138class LsstLogAdapter(LoggerAdapter): 

139 """A special logging adapter to provide log features for LSST code. 

140 

141 Expected to be instantiated initially by a call to `getLogger()`. 

142 

143 This class provides enhancements over `logging.Logger` that include: 

144 

145 * Methods for issuing trace and verbose level log messages. 

146 * Provision of a context manager to temporarily change the log level. 

147 * Attachment of logging level constants to the class to make it easier 

148 for a Task writer to access a specific log level without having to 

149 know the underlying logger class. 

150 """ 

151 

152 # Store logging constants in the class for convenience. This is not 

153 # something supported by Python loggers but can simplify some 

154 # logic if the logger is available. 

155 CRITICAL = logging.CRITICAL 

156 ERROR = logging.ERROR 

157 DEBUG = logging.DEBUG 

158 INFO = logging.INFO 

159 WARNING = logging.WARNING 

160 

161 # Python supports these but prefers they are not used. 

162 FATAL = logging.FATAL 

163 WARN = logging.WARN 

164 

165 # These are specific to Tasks 

166 TRACE = TRACE 

167 VERBOSE = VERBOSE 

168 

169 # The stack level to use when issuing log messages. For Python 3.11 

170 # this is generally 2 (this method and the internal infrastructure). 

171 # For older python we need one higher because of the extra indirection 

172 # via LoggingAdapter internals. 

173 _stacklevel = _calculate_base_stacklevel(2, 1) 

174 

175 @contextmanager 

176 def temporary_log_level(self, level: int | str) -> Generator: 

177 """Temporarily set the level of this logger. 

178 

179 Parameters 

180 ---------- 

181 level : `int` 

182 The new temporary log level. 

183 """ 

184 old = self.level 

185 self.setLevel(level) 

186 try: 

187 yield 

188 finally: 

189 self.setLevel(old) 

190 

191 @property 

192 def level(self) -> int: 

193 """Return current level of this logger (``int``).""" 

194 return self.logger.level 

195 

196 def getChild(self, name: str) -> LsstLogAdapter: 

197 """Get the named child logger. 

198 

199 Parameters 

200 ---------- 

201 name : `str` 

202 Name of the child relative to this logger. 

203 

204 Returns 

205 ------- 

206 child : `LsstLogAdapter` 

207 The child logger. 

208 """ 

209 return getLogger(name=name, logger=self.logger) 

210 

211 def _process_stacklevel(self, kwargs: dict[str, Any], offset: int = 0) -> int: 

212 # Return default stacklevel, taking into account kwargs[stacklevel]. 

213 stacklevel = self._stacklevel 

214 if "stacklevel" in kwargs: 

215 # External user expects stacklevel=1 to mean "report from their 

216 # line" but the code here is already trying to achieve that by 

217 # default. Therefore if an external stacklevel is specified we 

218 # adjust their stacklevel request by 1. 

219 stacklevel = stacklevel + kwargs.pop("stacklevel") - 1 

220 

221 # The offset can be used to indicate that we have to take into account 

222 # additional internal layers before calling python logging. 

223 return _calculate_base_stacklevel(stacklevel, offset) 

224 

225 def fatal(self, msg: str, *args: Any, **kwargs: Any) -> None: 

226 # Python does not provide this method in LoggerAdapter but does 

227 # not formally deprecate it in favor of critical() either. 

228 # Provide it without deprecation message for consistency with Python. 

229 # Have to adjust stacklevel on Python 3.10 and older to account 

230 # for call through self.critical. 

231 stacklevel = self._process_stacklevel(kwargs, offset=1) 

232 self.critical(msg, *args, **kwargs, stacklevel=stacklevel) 

233 

234 def verbose(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

235 """Issue a VERBOSE level log message. 

236 

237 Arguments are as for `logging.info`. 

238 ``VERBOSE`` is between ``DEBUG`` and ``INFO``. 

239 """ 

240 # There is no other way to achieve this other than a special logger 

241 # method. 

242 # Stacklevel is passed in so that the correct line is reported 

243 stacklevel = self._process_stacklevel(kwargs) 

244 self.log(VERBOSE, fmt, *args, **kwargs, stacklevel=stacklevel) 

245 

246 def trace(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

247 """Issue a TRACE level log message. 

248 

249 Arguments are as for `logging.info`. 

250 ``TRACE`` is lower than ``DEBUG``. 

251 """ 

252 # There is no other way to achieve this other than a special logger 

253 # method. 

254 stacklevel = self._process_stacklevel(kwargs) 

255 self.log(TRACE, fmt, *args, **kwargs, stacklevel=stacklevel) 

256 

257 def setLevel(self, level: int | str) -> None: 

258 """Set the level for the logger, trapping lsst.log values. 

259 

260 Parameters 

261 ---------- 

262 level : `int` 

263 The level to use. If the level looks too big to be a Python 

264 logging level it is assumed to be a lsst.log level. 

265 """ 

266 if isinstance(level, int) and level > logging.CRITICAL: 

267 self.logger.warning( 

268 "Attempting to set level to %d -- looks like an lsst.log level so scaling it accordingly.", 

269 level, 

270 ) 

271 level //= 1000 

272 

273 self.logger.setLevel(level) 

274 

275 @property 

276 def handlers(self) -> list[logging.Handler]: 

277 """Log handlers associated with this logger.""" 

278 return self.logger.handlers 

279 

280 def addHandler(self, handler: logging.Handler) -> None: 

281 """Add a handler to this logger. 

282 

283 The handler is forwarded to the underlying logger. 

284 """ 

285 self.logger.addHandler(handler) 

286 

287 def removeHandler(self, handler: logging.Handler) -> None: 

288 """Remove the given handler from the underlying logger.""" 

289 self.logger.removeHandler(handler) 

290 

291 

292def getLogger(name: str | None = None, logger: logging.Logger | None = None) -> LsstLogAdapter: 

293 """Get a logger compatible with LSST usage. 

294 

295 Parameters 

296 ---------- 

297 name : `str`, optional 

298 Name of the logger. Root logger if `None`. 

299 logger : `logging.Logger` or `LsstLogAdapter` 

300 If given the logger is converted to the relevant logger class. 

301 If ``name`` is given the logger is assumed to be a child of the 

302 supplied logger. 

303 

304 Returns 

305 ------- 

306 logger : `LsstLogAdapter` 

307 The relevant logger. 

308 

309 Notes 

310 ----- 

311 A `logging.LoggerAdapter` is used since it is easier to provide a more 

312 uniform interface than when using `logging.setLoggerClass`. An adapter 

313 can be wrapped around the root logger and the `~logging.setLoggerClass` 

314 will return the logger first given that name even if the name was 

315 used before the `~lsst.pipe.base.Task` was created. 

316 """ 

317 if not logger: 

318 logger = logging.getLogger(name) 

319 elif name: 

320 logger = logger.getChild(name) 

321 return LsstLogAdapter(logger, {}) 

322 

323 

324LsstLoggers = Union[logging.Logger, LsstLogAdapter] 

325 

326 

327def getTraceLogger(logger: str | LsstLoggers, trace_level: int) -> LsstLogAdapter: 

328 """Get a logger with the appropriate TRACE name. 

329 

330 Parameters 

331 ---------- 

332 logger : `logging.Logger` or `LsstLogAdapter` or `lsst.log.Log` or `str` 

333 A logger to be used to derive the new trace logger. Can be a logger 

334 object that supports the ``name`` property or a string. 

335 trace_level : `int` 

336 The trace level to use for the logger. 

337 

338 Returns 

339 ------- 

340 trace_logger : `LsstLogAdapter` 

341 A new trace logger. The name will be derived by pre-pending ``TRACEn.`` 

342 to the name of the supplied logger. If the root logger is given 

343 the returned logger will be named ``TRACEn``. 

344 """ 

345 name = getattr(logger, "name", str(logger)) 

346 trace_name = f"TRACE{trace_level}.{name}" if name else f"TRACE{trace_level}" 

347 return getLogger(trace_name) 

348 

349 

350class PeriodicLogger: 

351 """Issue log messages if a time threshold has elapsed. 

352 

353 This class can be used in long-running sections of code where it would 

354 be useful to issue a log message periodically to show that the 

355 algorithm is progressing. 

356 

357 Parameters 

358 ---------- 

359 logger : `logging.Logger` or `LsstLogAdapter` 

360 Logger to use when issuing a message. 

361 interval : `float` 

362 The minimum interval between log messages. If `None` the class 

363 default will be used. 

364 level : `int`, optional 

365 Log level to use when issuing messages. 

366 """ 

367 

368 LOGGING_INTERVAL = 600.0 

369 """Default interval between log messages.""" 

370 

371 def __init__(self, logger: LsstLoggers, interval: float | None = None, level: int = VERBOSE): 

372 self.logger = logger 

373 self.interval = interval if interval is not None else self.LOGGING_INTERVAL 

374 self.level = level 

375 self.next_log_time = time.time() + self.interval 

376 self.num_issued = 0 

377 

378 # The stacklevel we need to issue logs is determined by the type 

379 # of logger we have been given. A LoggerAdapter has an extra 

380 # level of indirection. In Python 3.11 the logging infrastructure 

381 # takes care to check for internal logging stack frames so there 

382 # is no need for a difference. 

383 self._stacklevel = _calculate_base_stacklevel(2, 1 if isinstance(self.logger, LoggerAdapter) else 0) 

384 

385 def log(self, msg: str, *args: Any) -> bool: 

386 """Issue a log message if the interval has elapsed. 

387 

388 Parameters 

389 ---------- 

390 msg : `str` 

391 Message to issue if the time has been exceeded. 

392 *args : Any 

393 Parameters to be passed to the log system. 

394 

395 Returns 

396 ------- 

397 issued : `bool` 

398 Returns `True` if a log message was sent to the logging system. 

399 Returns `False` if the interval has not yet elapsed. Returning 

400 `True` does not indicate whether the log message was in fact 

401 issued by the logging system. 

402 """ 

403 if (current_time := time.time()) > self.next_log_time: 

404 self.logger.log(self.level, msg, *args, stacklevel=self._stacklevel) 

405 self.next_log_time = current_time + self.interval 

406 self.num_issued += 1 

407 return True 

408 return False