Coverage for python/lsst/utils/logging.py: 49%

112 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-01 15:14 -0700

1# This file is part of utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ( 

15 "TRACE", 

16 "VERBOSE", 

17 "getLogger", 

18 "getTraceLogger", 

19 "LsstLogAdapter", 

20 "LsstLoggers", 

21 "PeriodicLogger", 

22 "trace_set_at", 

23) 

24 

25import logging 

26import sys 

27import time 

28from collections.abc import Generator 

29from contextlib import contextmanager 

30from logging import LoggerAdapter 

31from typing import Any, TypeAlias, Union 

32 

33try: 

34 import lsst.log.utils as logUtils 

35except ImportError: 

36 logUtils = None 

37 

38 

39# log level for trace (verbose debug). 

40TRACE = 5 

41logging.addLevelName(TRACE, "TRACE") 

42 

43# Verbose logging is midway between INFO and DEBUG. 

44VERBOSE = (logging.INFO + logging.DEBUG) // 2 

45logging.addLevelName(VERBOSE, "VERBOSE") 

46 

47 

48def _calculate_base_stacklevel(default: int, offset: int) -> int: 

49 """Calculate the default logging stacklevel to use. 

50 

51 Parameters 

52 ---------- 

53 default : `int` 

54 The stacklevel to use in Python 3.11 and newer where the only 

55 thing to take into account is the number of levels above the core 

56 Python logging infrastructure. 

57 offset : `int` 

58 The offset to apply for older Python implementations that need to 

59 take into account internal call stacks. 

60 

61 Returns 

62 ------- 

63 stacklevel : `int` 

64 The stack level to pass to internal logging APIs that should result 

65 in the log messages being reported in caller lines. 

66 

67 Notes 

68 ----- 

69 In Python 3.11 the logging infrastructure was fixed such that we no 

70 longer need to understand that a LoggerAdapter log messages need to 

71 have a stack level one higher than a Logger would need. ``stacklevel=1`` 

72 now always means "log from the caller's line" without the caller having 

73 to understand internal implementation details. 

74 """ 

75 stacklevel = default 

76 if sys.version_info < (3, 11, 0): 76 ↛ 77line 76 didn't jump to line 77, because the condition on line 76 was never true

77 stacklevel += offset 

78 return stacklevel 

79 

80 

81def trace_set_at(name: str, number: int) -> None: 

82 """Adjust logging level to display messages with the trace number being 

83 less than or equal to the provided value. 

84 

85 Parameters 

86 ---------- 

87 name : `str` 

88 Name of the logger. 

89 number : `int` 

90 The trace number threshold for display. 

91 

92 Examples 

93 -------- 

94 .. code-block:: python 

95 

96 lsst.utils.logging.trace_set_at("lsst.afw", 3) 

97 

98 This will set loggers ``TRACE0.lsst.afw`` to ``TRACE3.lsst.afw`` to 

99 ``DEBUG`` and ``TRACE4.lsst.afw`` and ``TRACE5.lsst.afw`` to ``INFO``. 

100 

101 Notes 

102 ----- 

103 Loggers ``TRACE0.`` to ``TRACE5.`` are set. All loggers above 

104 the specified threshold are set to ``INFO`` and those below the threshold 

105 are set to ``DEBUG``. The expectation is that ``TRACE`` loggers only 

106 issue ``DEBUG`` log messages. 

107 

108 If ``lsst.log`` is installed, this function will also call 

109 `lsst.log.utils.traceSetAt` to ensure that non-Python loggers are 

110 also configured correctly. 

111 """ 

112 for i in range(6): 

113 level = logging.INFO if i > number else logging.DEBUG 

114 getTraceLogger(name, i).setLevel(level) 

115 

116 # if lsst log is available also set the trace loggers there. 

117 if logUtils is not None: 

118 logUtils.traceSetAt(name, number) 

119 

120 

121class _F: 

122 """Format, supporting `str.format()` syntax. 

123 

124 Notes 

125 ----- 

126 This follows the recommendation from 

127 https://docs.python.org/3/howto/logging-cookbook.html#using-custom-message-objects 

128 """ 

129 

130 def __init__(self, fmt: str, /, *args: Any, **kwargs: Any): 

131 self.fmt = fmt 

132 self.args = args 

133 self.kwargs = kwargs 

134 

135 def __str__(self) -> str: 

136 return self.fmt.format(*self.args, **self.kwargs) 

137 

138 

139class LsstLogAdapter(LoggerAdapter): 

140 """A special logging adapter to provide log features for LSST code. 

141 

142 Expected to be instantiated initially by a call to `getLogger()`. 

143 

144 This class provides enhancements over `logging.Logger` that include: 

145 

146 * Methods for issuing trace and verbose level log messages. 

147 * Provision of a context manager to temporarily change the log level. 

148 * Attachment of logging level constants to the class to make it easier 

149 for a Task writer to access a specific log level without having to 

150 know the underlying logger class. 

151 """ 

152 

153 # Store logging constants in the class for convenience. This is not 

154 # something supported by Python loggers but can simplify some 

155 # logic if the logger is available. 

156 CRITICAL = logging.CRITICAL 

157 ERROR = logging.ERROR 

158 DEBUG = logging.DEBUG 

159 INFO = logging.INFO 

160 WARNING = logging.WARNING 

161 

162 # Python supports these but prefers they are not used. 

163 FATAL = logging.FATAL 

164 WARN = logging.WARN 

165 

166 # These are specific to Tasks 

167 TRACE = TRACE 

168 VERBOSE = VERBOSE 

169 

170 # The stack level to use when issuing log messages. For Python 3.11 

171 # this is generally 2 (this method and the internal infrastructure). 

172 # For older python we need one higher because of the extra indirection 

173 # via LoggingAdapter internals. 

174 _stacklevel = _calculate_base_stacklevel(2, 1) 

175 

176 @contextmanager 

177 def temporary_log_level(self, level: int | str) -> Generator: 

178 """Temporarily set the level of this logger. 

179 

180 Parameters 

181 ---------- 

182 level : `int` 

183 The new temporary log level. 

184 """ 

185 old = self.level 

186 self.setLevel(level) 

187 try: 

188 yield 

189 finally: 

190 self.setLevel(old) 

191 

192 @property 

193 def level(self) -> int: 

194 """Return current level of this logger (``int``).""" 

195 return self.logger.level 

196 

197 def getChild(self, name: str) -> LsstLogAdapter: 

198 """Get the named child logger. 

199 

200 Parameters 

201 ---------- 

202 name : `str` 

203 Name of the child relative to this logger. 

204 

205 Returns 

206 ------- 

207 child : `LsstLogAdapter` 

208 The child logger. 

209 """ 

210 return getLogger(name=name, logger=self.logger) 

211 

212 def _process_stacklevel(self, kwargs: dict[str, Any], offset: int = 0) -> int: 

213 # Return default stacklevel, taking into account kwargs[stacklevel]. 

214 stacklevel = self._stacklevel 

215 if "stacklevel" in kwargs: 

216 # External user expects stacklevel=1 to mean "report from their 

217 # line" but the code here is already trying to achieve that by 

218 # default. Therefore if an external stacklevel is specified we 

219 # adjust their stacklevel request by 1. 

220 stacklevel = stacklevel + kwargs.pop("stacklevel") - 1 

221 

222 # The offset can be used to indicate that we have to take into account 

223 # additional internal layers before calling python logging. 

224 return _calculate_base_stacklevel(stacklevel, offset) 

225 

226 def fatal(self, msg: str, *args: Any, **kwargs: Any) -> None: 

227 # Python does not provide this method in LoggerAdapter but does 

228 # not formally deprecate it in favor of critical() either. 

229 # Provide it without deprecation message for consistency with Python. 

230 # Have to adjust stacklevel on Python 3.10 and older to account 

231 # for call through self.critical. 

232 stacklevel = self._process_stacklevel(kwargs, offset=1) 

233 self.critical(msg, *args, **kwargs, stacklevel=stacklevel) 

234 

235 def verbose(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

236 """Issue a VERBOSE level log message. 

237 

238 Arguments are as for `logging.info`. 

239 ``VERBOSE`` is between ``DEBUG`` and ``INFO``. 

240 

241 Parameters 

242 ---------- 

243 fmt : `str` 

244 Log message. 

245 *args : `~typing.Any` 

246 Parameters references by log message. 

247 **kwargs : `~typing.Any` 

248 Parameters forwarded to `log`. 

249 """ 

250 # There is no other way to achieve this other than a special logger 

251 # method. 

252 # Stacklevel is passed in so that the correct line is reported 

253 stacklevel = self._process_stacklevel(kwargs) 

254 self.log(VERBOSE, fmt, *args, **kwargs, stacklevel=stacklevel) 

255 

256 def trace(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

257 """Issue a TRACE level log message. 

258 

259 Arguments are as for `logging.info`. 

260 ``TRACE`` is lower than ``DEBUG``. 

261 

262 Parameters 

263 ---------- 

264 fmt : `str` 

265 Log message. 

266 *args : `~typing.Any` 

267 Parameters references by log message. 

268 **kwargs : `~typing.Any` 

269 Parameters forwarded to `log`. 

270 """ 

271 # There is no other way to achieve this other than a special logger 

272 # method. 

273 stacklevel = self._process_stacklevel(kwargs) 

274 self.log(TRACE, fmt, *args, **kwargs, stacklevel=stacklevel) 

275 

276 def setLevel(self, level: int | str) -> None: 

277 """Set the level for the logger, trapping lsst.log values. 

278 

279 Parameters 

280 ---------- 

281 level : `int` 

282 The level to use. If the level looks too big to be a Python 

283 logging level it is assumed to be a lsst.log level. 

284 """ 

285 if isinstance(level, int) and level > logging.CRITICAL: 

286 self.logger.warning( 

287 "Attempting to set level to %d -- looks like an lsst.log level so scaling it accordingly.", 

288 level, 

289 ) 

290 level //= 1000 

291 

292 self.logger.setLevel(level) 

293 

294 @property 

295 def handlers(self) -> list[logging.Handler]: 

296 """Log handlers associated with this logger.""" 

297 return self.logger.handlers 

298 

299 def addHandler(self, handler: logging.Handler) -> None: 

300 """Add a handler to this logger. 

301 

302 Parameters 

303 ---------- 

304 handler : `logging.Handler` 

305 Handler to add. The handler is forwarded to the underlying logger. 

306 """ 

307 self.logger.addHandler(handler) 

308 

309 def removeHandler(self, handler: logging.Handler) -> None: 

310 """Remove the given handler from the underlying logger. 

311 

312 Parameters 

313 ---------- 

314 handler : `logging.Handler` 

315 Handler to remove. 

316 """ 

317 self.logger.removeHandler(handler) 

318 

319 

320def getLogger(name: str | None = None, logger: logging.Logger | None = None) -> LsstLogAdapter: 

321 """Get a logger compatible with LSST usage. 

322 

323 Parameters 

324 ---------- 

325 name : `str`, optional 

326 Name of the logger. Root logger if `None`. 

327 logger : `logging.Logger` or `LsstLogAdapter` 

328 If given the logger is converted to the relevant logger class. 

329 If ``name`` is given the logger is assumed to be a child of the 

330 supplied logger. 

331 

332 Returns 

333 ------- 

334 logger : `LsstLogAdapter` 

335 The relevant logger. 

336 

337 Notes 

338 ----- 

339 A `logging.LoggerAdapter` is used since it is easier to provide a more 

340 uniform interface than when using `logging.setLoggerClass`. An adapter 

341 can be wrapped around the root logger and the `~logging.setLoggerClass` 

342 will return the logger first given that name even if the name was 

343 used before the `~lsst.pipe.base.Task` was created. 

344 """ 

345 if not logger: 

346 logger = logging.getLogger(name) 

347 elif name: 

348 logger = logger.getChild(name) 

349 return LsstLogAdapter(logger, {}) 

350 

351 

352LsstLoggers: TypeAlias = Union[logging.Logger, LsstLogAdapter] 

353 

354 

355def getTraceLogger(logger: str | LsstLoggers, trace_level: int) -> LsstLogAdapter: 

356 """Get a logger with the appropriate TRACE name. 

357 

358 Parameters 

359 ---------- 

360 logger : `logging.Logger` or `LsstLogAdapter` or `lsst.log.Log` or `str` 

361 A logger to be used to derive the new trace logger. Can be a logger 

362 object that supports the ``name`` property or a string. 

363 trace_level : `int` 

364 The trace level to use for the logger. 

365 

366 Returns 

367 ------- 

368 trace_logger : `LsstLogAdapter` 

369 A new trace logger. The name will be derived by pre-pending ``TRACEn.`` 

370 to the name of the supplied logger. If the root logger is given 

371 the returned logger will be named ``TRACEn``. 

372 """ 

373 name = getattr(logger, "name", str(logger)) 

374 trace_name = f"TRACE{trace_level}.{name}" if name else f"TRACE{trace_level}" 

375 return getLogger(trace_name) 

376 

377 

378class PeriodicLogger: 

379 """Issue log messages if a time threshold has elapsed. 

380 

381 This class can be used in long-running sections of code where it would 

382 be useful to issue a log message periodically to show that the 

383 algorithm is progressing. 

384 

385 Parameters 

386 ---------- 

387 logger : `logging.Logger` or `LsstLogAdapter` 

388 Logger to use when issuing a message. 

389 interval : `float` 

390 The minimum interval between log messages. If `None` the class 

391 default will be used. 

392 level : `int`, optional 

393 Log level to use when issuing messages. 

394 """ 

395 

396 LOGGING_INTERVAL = 600.0 

397 """Default interval between log messages.""" 

398 

399 def __init__(self, logger: LsstLoggers, interval: float | None = None, level: int = VERBOSE): 

400 self.logger = logger 

401 self.interval = interval if interval is not None else self.LOGGING_INTERVAL 

402 self.level = level 

403 self.next_log_time = time.time() + self.interval 

404 self.num_issued = 0 

405 

406 # The stacklevel we need to issue logs is determined by the type 

407 # of logger we have been given. A LoggerAdapter has an extra 

408 # level of indirection. In Python 3.11 the logging infrastructure 

409 # takes care to check for internal logging stack frames so there 

410 # is no need for a difference. 

411 self._stacklevel = _calculate_base_stacklevel(2, 1 if isinstance(self.logger, LoggerAdapter) else 0) 

412 

413 def log(self, msg: str, *args: Any) -> bool: 

414 """Issue a log message if the interval has elapsed. 

415 

416 Parameters 

417 ---------- 

418 msg : `str` 

419 Message to issue if the time has been exceeded. 

420 *args : Any 

421 Parameters to be passed to the log system. 

422 

423 Returns 

424 ------- 

425 issued : `bool` 

426 Returns `True` if a log message was sent to the logging system. 

427 Returns `False` if the interval has not yet elapsed. Returning 

428 `True` does not indicate whether the log message was in fact 

429 issued by the logging system. 

430 """ 

431 if (current_time := time.time()) > self.next_log_time: 

432 self.logger.log(self.level, msg, *args, stacklevel=self._stacklevel) 

433 self.next_log_time = current_time + self.interval 

434 self.num_issued += 1 

435 return True 

436 return False