Coverage for python/lsst/utils/logging.py: 49%

112 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-19 11:15 +0000

1# This file is part of utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12from __future__ import annotations 

13 

14__all__ = ( 

15 "TRACE", 

16 "VERBOSE", 

17 "getLogger", 

18 "getTraceLogger", 

19 "LsstLogAdapter", 

20 "PeriodicLogger", 

21 "trace_set_at", 

22) 

23 

24import logging 

25import sys 

26import time 

27from collections.abc import Generator 

28from contextlib import contextmanager 

29from logging import LoggerAdapter 

30from typing import Any, Union 

31 

32try: 

33 import lsst.log.utils as logUtils 

34except ImportError: 

35 logUtils = None 

36 

37 

38# log level for trace (verbose debug). 

39TRACE = 5 

40logging.addLevelName(TRACE, "TRACE") 

41 

42# Verbose logging is midway between INFO and DEBUG. 

43VERBOSE = (logging.INFO + logging.DEBUG) // 2 

44logging.addLevelName(VERBOSE, "VERBOSE") 

45 

46 

47def _calculate_base_stacklevel(default: int, offset: int) -> int: 

48 """Calculate the default logging stacklevel to use. 

49 

50 Parameters 

51 ---------- 

52 default : `int` 

53 The stacklevel to use in Python 3.11 and newer where the only 

54 thing to take into account is the number of levels above the core 

55 Python logging infrastructure. 

56 offset : `int` 

57 The offset to apply for older Python implementations that need to 

58 take into account internal call stacks. 

59 

60 Returns 

61 ------- 

62 stacklevel : `int` 

63 The stack level to pass to internal logging APIs that should result 

64 in the log messages being reported in caller lines. 

65 

66 Notes 

67 ----- 

68 In Python 3.11 the logging infrastructure was fixed such that we no 

69 longer need to understand that a LoggerAdapter log messages need to 

70 have a stack level one higher than a Logger would need. ``stacklevel=1`` 

71 now always means "log from the caller's line" without the caller having 

72 to understand internal implementation details. 

73 """ 

74 stacklevel = default 

75 if sys.version_info < (3, 11, 0): 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true

76 stacklevel += offset 

77 return stacklevel 

78 

79 

80def trace_set_at(name: str, number: int) -> None: 

81 """Adjust logging level to display messages with the trace number being 

82 less than or equal to the provided value. 

83 

84 Parameters 

85 ---------- 

86 name : `str` 

87 Name of the logger. 

88 number : `int` 

89 The trace number threshold for display. 

90 

91 Examples 

92 -------- 

93 .. code-block:: python 

94 

95 lsst.utils.logging.trace_set_at("lsst.afw", 3) 

96 

97 This will set loggers ``TRACE0.lsst.afw`` to ``TRACE3.lsst.afw`` to 

98 ``DEBUG`` and ``TRACE4.lsst.afw`` and ``TRACE5.lsst.afw`` to ``INFO``. 

99 

100 Notes 

101 ----- 

102 Loggers ``TRACE0.`` to ``TRACE5.`` are set. All loggers above 

103 the specified threshold are set to ``INFO`` and those below the threshold 

104 are set to ``DEBUG``. The expectation is that ``TRACE`` loggers only 

105 issue ``DEBUG`` log messages. 

106 

107 If ``lsst.log`` is installed, this function will also call 

108 `lsst.log.utils.traceSetAt` to ensure that non-Python loggers are 

109 also configured correctly. 

110 """ 

111 for i in range(6): 

112 level = logging.INFO if i > number else logging.DEBUG 

113 getTraceLogger(name, i).setLevel(level) 

114 

115 # if lsst log is available also set the trace loggers there. 

116 if logUtils is not None: 

117 logUtils.traceSetAt(name, number) 

118 

119 

120class _F: 

121 """Format, supporting `str.format()` syntax. 

122 

123 Notes 

124 ----- 

125 This follows the recommendation from 

126 https://docs.python.org/3/howto/logging-cookbook.html#using-custom-message-objects 

127 """ 

128 

129 def __init__(self, fmt: str, /, *args: Any, **kwargs: Any): 

130 self.fmt = fmt 

131 self.args = args 

132 self.kwargs = kwargs 

133 

134 def __str__(self) -> str: 

135 return self.fmt.format(*self.args, **self.kwargs) 

136 

137 

138class LsstLogAdapter(LoggerAdapter): 

139 """A special logging adapter to provide log features for LSST code. 

140 

141 Expected to be instantiated initially by a call to `getLogger()`. 

142 

143 This class provides enhancements over `logging.Logger` that include: 

144 

145 * Methods for issuing trace and verbose level log messages. 

146 * Provision of a context manager to temporarily change the log level. 

147 * Attachment of logging level constants to the class to make it easier 

148 for a Task writer to access a specific log level without having to 

149 know the underlying logger class. 

150 """ 

151 

152 # Store logging constants in the class for convenience. This is not 

153 # something supported by Python loggers but can simplify some 

154 # logic if the logger is available. 

155 CRITICAL = logging.CRITICAL 

156 ERROR = logging.ERROR 

157 DEBUG = logging.DEBUG 

158 INFO = logging.INFO 

159 WARNING = logging.WARNING 

160 

161 # Python supports these but prefers they are not used. 

162 FATAL = logging.FATAL 

163 WARN = logging.WARN 

164 

165 # These are specific to Tasks 

166 TRACE = TRACE 

167 VERBOSE = VERBOSE 

168 

169 # The stack level to use when issuing log messages. For Python 3.11 

170 # this is generally 2 (this method and the internal infrastructure). 

171 # For older python we need one higher because of the extra indirection 

172 # via LoggingAdapter internals. 

173 _stacklevel = _calculate_base_stacklevel(2, 1) 

174 

175 @contextmanager 

176 def temporary_log_level(self, level: int | str) -> Generator: 

177 """Temporarily set the level of this logger. 

178 

179 Parameters 

180 ---------- 

181 level : `int` 

182 The new temporary log level. 

183 """ 

184 old = self.level 

185 self.setLevel(level) 

186 try: 

187 yield 

188 finally: 

189 self.setLevel(old) 

190 

191 @property 

192 def level(self) -> int: 

193 """Return current level of this logger (``int``).""" 

194 return self.logger.level 

195 

196 def getChild(self, name: str) -> LsstLogAdapter: 

197 """Get the named child logger. 

198 

199 Parameters 

200 ---------- 

201 name : `str` 

202 Name of the child relative to this logger. 

203 

204 Returns 

205 ------- 

206 child : `LsstLogAdapter` 

207 The child logger. 

208 """ 

209 return getLogger(name=name, logger=self.logger) 

210 

211 def _process_stacklevel(self, kwargs: dict[str, Any], offset: int = 0) -> int: 

212 # Return default stacklevel, taking into account kwargs[stacklevel]. 

213 stacklevel = self._stacklevel 

214 if "stacklevel" in kwargs: 

215 # External user expects stacklevel=1 to mean "report from their 

216 # line" but the code here is already trying to achieve that by 

217 # default. Therefore if an external stacklevel is specified we 

218 # adjust their stacklevel request by 1. 

219 stacklevel = stacklevel + kwargs.pop("stacklevel") - 1 

220 

221 # The offset can be used to indicate that we have to take into account 

222 # additional internal layers before calling python logging. 

223 return _calculate_base_stacklevel(stacklevel, offset) 

224 

225 def fatal(self, msg: str, *args: Any, **kwargs: Any) -> None: 

226 # Python does not provide this method in LoggerAdapter but does 

227 # not formally deprecate it in favor of critical() either. 

228 # Provide it without deprecation message for consistency with Python. 

229 # Have to adjust stacklevel on Python 3.10 and older to account 

230 # for call through self.critical. 

231 stacklevel = self._process_stacklevel(kwargs, offset=1) 

232 self.critical(msg, *args, **kwargs, stacklevel=stacklevel) 

233 

234 def verbose(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

235 """Issue a VERBOSE level log message. 

236 

237 Arguments are as for `logging.info`. 

238 ``VERBOSE`` is between ``DEBUG`` and ``INFO``. 

239 

240 Parameters 

241 ---------- 

242 fmt : `str` 

243 Log message. 

244 *args : `~typing.Any` 

245 Parameters references by log message. 

246 **kwargs : `~typing.Any` 

247 Parameters forwarded to `log`. 

248 """ 

249 # There is no other way to achieve this other than a special logger 

250 # method. 

251 # Stacklevel is passed in so that the correct line is reported 

252 stacklevel = self._process_stacklevel(kwargs) 

253 self.log(VERBOSE, fmt, *args, **kwargs, stacklevel=stacklevel) 

254 

255 def trace(self, fmt: str, *args: Any, **kwargs: Any) -> None: 

256 """Issue a TRACE level log message. 

257 

258 Arguments are as for `logging.info`. 

259 ``TRACE`` is lower than ``DEBUG``. 

260 

261 Parameters 

262 ---------- 

263 fmt : `str` 

264 Log message. 

265 *args : `~typing.Any` 

266 Parameters references by log message. 

267 **kwargs : `~typing.Any` 

268 Parameters forwarded to `log`. 

269 """ 

270 # There is no other way to achieve this other than a special logger 

271 # method. 

272 stacklevel = self._process_stacklevel(kwargs) 

273 self.log(TRACE, fmt, *args, **kwargs, stacklevel=stacklevel) 

274 

275 def setLevel(self, level: int | str) -> None: 

276 """Set the level for the logger, trapping lsst.log values. 

277 

278 Parameters 

279 ---------- 

280 level : `int` 

281 The level to use. If the level looks too big to be a Python 

282 logging level it is assumed to be a lsst.log level. 

283 """ 

284 if isinstance(level, int) and level > logging.CRITICAL: 

285 self.logger.warning( 

286 "Attempting to set level to %d -- looks like an lsst.log level so scaling it accordingly.", 

287 level, 

288 ) 

289 level //= 1000 

290 

291 self.logger.setLevel(level) 

292 

293 @property 

294 def handlers(self) -> list[logging.Handler]: 

295 """Log handlers associated with this logger.""" 

296 return self.logger.handlers 

297 

298 def addHandler(self, handler: logging.Handler) -> None: 

299 """Add a handler to this logger. 

300 

301 Parameters 

302 ---------- 

303 handler : `logging.Handler` 

304 Handler to add. The handler is forwarded to the underlying logger. 

305 """ 

306 self.logger.addHandler(handler) 

307 

308 def removeHandler(self, handler: logging.Handler) -> None: 

309 """Remove the given handler from the underlying logger. 

310 

311 Parameters 

312 ---------- 

313 handler : `logging.Handler` 

314 Handler to remove. 

315 """ 

316 self.logger.removeHandler(handler) 

317 

318 

319def getLogger(name: str | None = None, logger: logging.Logger | None = None) -> LsstLogAdapter: 

320 """Get a logger compatible with LSST usage. 

321 

322 Parameters 

323 ---------- 

324 name : `str`, optional 

325 Name of the logger. Root logger if `None`. 

326 logger : `logging.Logger` or `LsstLogAdapter` 

327 If given the logger is converted to the relevant logger class. 

328 If ``name`` is given the logger is assumed to be a child of the 

329 supplied logger. 

330 

331 Returns 

332 ------- 

333 logger : `LsstLogAdapter` 

334 The relevant logger. 

335 

336 Notes 

337 ----- 

338 A `logging.LoggerAdapter` is used since it is easier to provide a more 

339 uniform interface than when using `logging.setLoggerClass`. An adapter 

340 can be wrapped around the root logger and the `~logging.setLoggerClass` 

341 will return the logger first given that name even if the name was 

342 used before the `~lsst.pipe.base.Task` was created. 

343 """ 

344 if not logger: 

345 logger = logging.getLogger(name) 

346 elif name: 

347 logger = logger.getChild(name) 

348 return LsstLogAdapter(logger, {}) 

349 

350 

351LsstLoggers = Union[logging.Logger, LsstLogAdapter] 

352 

353 

354def getTraceLogger(logger: str | LsstLoggers, trace_level: int) -> LsstLogAdapter: 

355 """Get a logger with the appropriate TRACE name. 

356 

357 Parameters 

358 ---------- 

359 logger : `logging.Logger` or `LsstLogAdapter` or `lsst.log.Log` or `str` 

360 A logger to be used to derive the new trace logger. Can be a logger 

361 object that supports the ``name`` property or a string. 

362 trace_level : `int` 

363 The trace level to use for the logger. 

364 

365 Returns 

366 ------- 

367 trace_logger : `LsstLogAdapter` 

368 A new trace logger. The name will be derived by pre-pending ``TRACEn.`` 

369 to the name of the supplied logger. If the root logger is given 

370 the returned logger will be named ``TRACEn``. 

371 """ 

372 name = getattr(logger, "name", str(logger)) 

373 trace_name = f"TRACE{trace_level}.{name}" if name else f"TRACE{trace_level}" 

374 return getLogger(trace_name) 

375 

376 

377class PeriodicLogger: 

378 """Issue log messages if a time threshold has elapsed. 

379 

380 This class can be used in long-running sections of code where it would 

381 be useful to issue a log message periodically to show that the 

382 algorithm is progressing. 

383 

384 Parameters 

385 ---------- 

386 logger : `logging.Logger` or `LsstLogAdapter` 

387 Logger to use when issuing a message. 

388 interval : `float` 

389 The minimum interval between log messages. If `None` the class 

390 default will be used. 

391 level : `int`, optional 

392 Log level to use when issuing messages. 

393 """ 

394 

395 LOGGING_INTERVAL = 600.0 

396 """Default interval between log messages.""" 

397 

398 def __init__(self, logger: LsstLoggers, interval: float | None = None, level: int = VERBOSE): 

399 self.logger = logger 

400 self.interval = interval if interval is not None else self.LOGGING_INTERVAL 

401 self.level = level 

402 self.next_log_time = time.time() + self.interval 

403 self.num_issued = 0 

404 

405 # The stacklevel we need to issue logs is determined by the type 

406 # of logger we have been given. A LoggerAdapter has an extra 

407 # level of indirection. In Python 3.11 the logging infrastructure 

408 # takes care to check for internal logging stack frames so there 

409 # is no need for a difference. 

410 self._stacklevel = _calculate_base_stacklevel(2, 1 if isinstance(self.logger, LoggerAdapter) else 0) 

411 

412 def log(self, msg: str, *args: Any) -> bool: 

413 """Issue a log message if the interval has elapsed. 

414 

415 Parameters 

416 ---------- 

417 msg : `str` 

418 Message to issue if the time has been exceeded. 

419 *args : Any 

420 Parameters to be passed to the log system. 

421 

422 Returns 

423 ------- 

424 issued : `bool` 

425 Returns `True` if a log message was sent to the logging system. 

426 Returns `False` if the interval has not yet elapsed. Returning 

427 `True` does not indicate whether the log message was in fact 

428 issued by the logging system. 

429 """ 

430 if (current_time := time.time()) > self.next_log_time: 

431 self.logger.log(self.level, msg, *args, stacklevel=self._stacklevel) 

432 self.next_log_time = current_time + self.interval 

433 self.num_issued += 1 

434 return True 

435 return False