Coverage for python/lsst/daf/butler/tests/cliLogTestBase.py: 32%

136 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for the daf_butler CliLog utility. 

29 

30Code is implemented in 

31daf_butler but some only runs if lsst.log.Log can be imported so these parts of 

32it can't be tested there because daf_butler does not directly depend on 

33lsst.log, and only uses it if it has been setup by another package. 

34""" 

35 

36from __future__ import annotations 

37 

38import logging 

39import os 

40import re 

41import subprocess 

42import tempfile 

43import unittest 

44from collections import namedtuple 

45from collections.abc import Callable 

46from functools import partial 

47from io import StringIO 

48from logging import DEBUG, INFO, WARNING 

49from typing import TYPE_CHECKING, Any 

50 

51import click 

52from lsst.daf.butler.cli.butler import cli as butlerCli 

53from lsst.daf.butler.cli.cliLog import CliLog 

54from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, command_test_env 

55from lsst.daf.butler.logging import ButlerLogRecords 

56from lsst.utils.logging import TRACE 

57 

58try: 

59 import lsst.log as lsstLog 

60 

61 lsstLog_INFO = lsstLog.INFO 

62 lsstLog_DEBUG = lsstLog.DEBUG 

63 lsstLog_WARN = lsstLog.WARN 

64except ModuleNotFoundError: 

65 lsstLog = None 

66 lsstLog_INFO = 0 

67 lsstLog_DEBUG = 0 

68 lsstLog_WARN = 0 

69 

70 

71@click.command() 

72@click.option("--expected-pyroot-level", type=int) 

73@click.option("--expected-pylsst-level", type=int) 

74@click.option("--expected-pybutler-level", type=int) 

75@click.option("--expected-lsstroot-level", type=int) 

76@click.option("--expected-lsstbutler-level", type=int) 

77@click.option("--expected-lsstx-level", type=int) 

78def command_log_settings_test( # numpydoc ignore=PR01 

79 expected_pyroot_level: str, 

80 expected_pylsst_level: str, 

81 expected_pybutler_level: str, 

82 expected_lsstroot_level: str, 

83 expected_lsstbutler_level: str, 

84 expected_lsstx_level: str, 

85) -> None: 

86 """Test command-line log settings.""" 

87 LogLevel = namedtuple("LogLevel", ("expected", "actual", "name")) 

88 

89 logLevels = [ 

90 LogLevel(expected_pyroot_level, logging.getLogger().level, "pyRoot"), 

91 LogLevel(expected_pylsst_level, logging.getLogger("lsst").getEffectiveLevel(), "pyLsst"), 

92 LogLevel( 

93 expected_pybutler_level, logging.getLogger("lsst.daf.butler").getEffectiveLevel(), "pyButler" 

94 ), 

95 LogLevel(expected_lsstx_level, logging.getLogger("lsstx").getEffectiveLevel(), "pyLsstx"), 

96 ] 

97 if lsstLog is not None: 

98 logLevels.extend( 

99 [ 

100 LogLevel(expected_lsstroot_level, lsstLog.getLogger("lsst").getEffectiveLevel(), "lsstRoot"), 

101 LogLevel( 

102 expected_lsstbutler_level, 

103 lsstLog.getLogger("lsst.daf.butler").getEffectiveLevel(), 

104 "lsstButler", 

105 ), 

106 ] 

107 ) 

108 for expected, actual, name in logLevels: 

109 if expected != actual: 

110 raise ( 

111 click.ClickException(message=f"expected {name} level to be {expected!r}, actual:{actual!r}") 

112 ) 

113 

114 

115class CliLogTestBase: 

116 """Tests log initialization, reset, and setting log levels.""" 

117 

118 if TYPE_CHECKING: 

119 assertEqual: Callable 

120 assertIn: Callable 

121 assertTrue: Callable 

122 assertFalse: Callable 

123 assertGreater: Callable 

124 subTest: Callable 

125 assertNotIn: Callable 

126 

127 def setUp(self) -> None: 

128 self.runner = LogCliRunner() 

129 

130 class PythonLogger: 

131 """Keeps track of log level of a component and number of handlers 

132 attached to it at the time this object was initialized. 

133 

134 Parameters 

135 ---------- 

136 component : `str` or `None` 

137 The logger name. 

138 """ 

139 

140 def __init__(self, component: str | None) -> None: 

141 self.logger = logging.getLogger(component) 

142 self.initialLevel = self.logger.level 

143 

144 class LsstLogger: 

145 """Keeps track of log level for a component at the time this object was 

146 initialized. 

147 

148 Parameters 

149 ---------- 

150 component : `str` or `None` 

151 The logger name. 

152 """ 

153 

154 def __init__(self, component: str) -> None: 

155 self.logger = lsstLog.getLogger(component) if lsstLog else None 

156 self.initialLevel = self.logger.getLevel() if lsstLog else None 

157 

158 def runTest(self, cmd: Callable) -> None: 

159 """Test that the log context manager works with the butler cli. 

160 

161 Parameters 

162 ---------- 

163 cmd : `~collections.abc.Callable` 

164 The command to run. 

165 

166 Notes 

167 ----- 

168 Tests that it will initialize the logging system according to cli 

169 inputs for the duration of the command execution and resets the logging 

170 system to its previous state or expected state when command execution 

171 finishes. 

172 """ 

173 pyRoot = self.PythonLogger(None) 

174 pyButler = self.PythonLogger("lsst.daf.butler") 

175 pyLsstRoot = self.PythonLogger("lsst") 

176 lsstRoot = self.LsstLogger("") 

177 lsstButler = self.LsstLogger("lsst.daf.butler") 

178 

179 with command_test_env( 

180 self.runner, "lsst.daf.butler.tests.cliLogTestBase", "command-log-settings-test" 

181 ): 

182 result = cmd() 

183 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

184 

185 # The test environment may have changed the python root logger 

186 # so we can not assume it will be WARNING. 

187 self.assertEqual(pyRoot.logger.level, logging.getLogger().getEffectiveLevel()) 

188 self.assertEqual(pyLsstRoot.logger.level, logging.INFO) 

189 self.assertEqual(pyButler.logger.level, pyButler.initialLevel) 

190 if lsstLog is not None: 

191 self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO) 

192 # lsstLogLevel can either be the initial level, or uninitialized or 

193 # the defined default value. 

194 expectedLsstLogLevel = ( 

195 (lsstButler.initialLevel,) 

196 if lsstButler.initialLevel != -1 

197 else (-1, CliLog.defaultLsstLogLevel) 

198 ) 

199 self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel) 

200 

201 def test_butlerCliLog(self) -> None: 

202 """Test that the log context manager works with the butler cli to 

203 initialize the logging system according to cli inputs for the duration 

204 of the command execution and resets the logging system to its previous 

205 state or expected state when command execution finishes. 

206 """ 

207 # Run with two different log level settings. 

208 log_levels = ( 

209 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

210 # lsstroot, lsstbutler, lsstx 

211 ( 

212 "WARNING", 

213 "lsst.daf.butler=DEBUG", 

214 WARNING, 

215 WARNING, 

216 DEBUG, 

217 lsstLog_WARN, 

218 lsstLog_DEBUG, 

219 WARNING, 

220 ), 

221 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, WARNING), 

222 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, DEBUG), 

223 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

224 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, DEBUG), 

225 ) 

226 

227 self._test_levels(log_levels) 

228 

229 # Check that the environment variable can set additional roots. 

230 log_levels = ( 

231 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

232 # lsstroot, lsstbutler, lsstx 

233 ( 

234 "WARNING", 

235 "lsst.daf.butler=DEBUG", 

236 WARNING, 

237 WARNING, 

238 DEBUG, 

239 lsstLog_WARN, 

240 lsstLog_DEBUG, 

241 WARNING, 

242 ), 

243 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

244 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, INFO), 

245 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

246 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, INFO), 

247 ) 

248 

249 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_ROOT_LOGGER": "lsstx"}): 

250 self._test_levels(log_levels) 

251 

252 def _test_levels(self, log_levels: tuple[tuple[str, str, int, int, int, Any, Any, int], ...]) -> None: 

253 for level1, level2, x_pyroot, x_pylsst, x_pybutler, x_lsstroot, x_lsstbutler, x_lsstx in log_levels: 

254 with self.subTest("Test different log levels", level1=level1, level2=level2): 

255 self.runTest( 

256 partial( 

257 self.runner.invoke, 

258 butlerCli, 

259 [ 

260 "--log-level", 

261 level1, 

262 "--log-level", 

263 level2, 

264 "command-log-settings-test", 

265 "--expected-pyroot-level", 

266 x_pyroot, 

267 "--expected-pylsst-level", 

268 x_pylsst, 

269 "--expected-pybutler-level", 

270 x_pybutler, 

271 "--expected-lsstroot-level", 

272 x_lsstroot, 

273 "--expected-lsstbutler-level", 

274 x_lsstbutler, 

275 "--expected-lsstx-level", 

276 x_lsstx, 

277 ], 

278 ) 

279 ) 

280 

281 def test_helpLogReset(self) -> None: 

282 """Verify that when a command does not execute, like when the help menu 

283 is printed instead, that CliLog is still reset. 

284 """ 

285 self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"])) 

286 

287 def testLongLog(self) -> None: 

288 """Verify the timestamp is in the log messages when the --long-log 

289 flag is set. 

290 """ 

291 # When longlog=True, loglines start with the log level and a 

292 # timestamp with the following format: 

293 # "year-month-day T hour-minute-second.millisecond-zoneoffset" 

294 # For example: "DEBUG 2020-10-28T10:20:31-07:00 ..."" 

295 # The log level name can change, we verify there is an all 

296 # caps word there but do not verify the word. We do not verify 

297 # the rest of the log string, assume that if the timestamp is 

298 # in the string that the rest of the string will appear as 

299 # expected. 

300 timestampRegex = re.compile( 

301 r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})" 

302 "?([-,+][01][0-9]:[034][05]|Z) .*" 

303 ) 

304 

305 # When longlog=False, log lines start with the module name and 

306 # log level, for example: 

307 # lsst.daf.butler.config DEBUG: ... 

308 modulesRegex = re.compile(r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*") 

309 

310 with self.runner.isolated_filesystem(): 

311 for longlog in (True, False): 

312 # The pytest log handler interferes with the log configuration 

313 # settings set up by initLog -- therefore test by using 

314 # a subprocess. 

315 args: tuple[str, ...] 

316 if longlog: 

317 args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here") 

318 else: 

319 args = ("butler", "--log-level", "DEBUG", "create", "here") 

320 result = subprocess.run(args, capture_output=True) 

321 # There are cases where the newlines are stripped from the log 

322 # output (like in Jenkins), since we can't depend on newlines 

323 # in log output they are removed here from test output. 

324 output = StringIO(result.stderr.decode().replace("\n", " ")) 

325 startedWithTimestamp = any(timestampRegex.match(line) for line in output.readlines()) 

326 output.seek(0) 

327 startedWithModule = any(modulesRegex.match(line) for line in output.readlines()) 

328 if longlog: 

329 self.assertTrue( 

330 startedWithTimestamp, msg=f"did not find timestamp in: \n{output.getvalue()}" 

331 ) 

332 self.assertFalse( 

333 startedWithModule, msg=f"found lines starting with module in: \n{output.getvalue()}" 

334 ) 

335 else: 

336 self.assertFalse(startedWithTimestamp, msg=f"found timestamp in: \n{output.getvalue()}") 

337 self.assertTrue( 

338 startedWithModule, 

339 msg=f"did not find lines starting with module in: \n{output.getvalue()}", 

340 ) 

341 

342 def testFileLogging(self) -> None: 

343 """Test --log-file option.""" 

344 with self.runner.isolated_filesystem(): 

345 for i, suffix in enumerate([".json", ".log"]): 

346 # Get a temporary file name and immediately close it 

347 fd = tempfile.NamedTemporaryFile(suffix=suffix) 

348 filename = fd.name 

349 fd.close() 

350 

351 args = ( 

352 "--log-level", 

353 "DEBUG", 

354 "--log-file", 

355 filename, 

356 "--log-label", 

357 "k1=v1,k2=v2", 

358 "--log-label", 

359 "k3=v3", 

360 "create", 

361 f"here{i}", 

362 ) 

363 

364 result = self.runner.invoke(butlerCli, args) 

365 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

366 

367 # Record to test. Test one in the middle that we know is 

368 # a DEBUG message. The first message might come from 

369 # python itself since warnings are redirected to log 

370 # messages. 

371 num = 4 

372 

373 n_records = 5 

374 if suffix == ".json": 

375 records = ButlerLogRecords.from_file(filename) 

376 self.assertGreater(len(records), num) 

377 self.assertEqual(records[num].levelname, "DEBUG", str(records[num])) 

378 self.assertEqual(records[0].MDC, dict(K1="v1", K2="v2", K3="v3")) 

379 

380 self.assertGreater(len(records), n_records) 

381 else: 

382 with open(filename) as filed: 

383 records_text = filed.readlines() 

384 self.assertGreater(len(records_text), num) 

385 self.assertIn("DEBUG", records_text[num], str(records_text[num])) 

386 self.assertNotIn("{", records_text[num], str(records_text[num])) 

387 

388 self.assertGreater(len(records), n_records) 

389 

390 def testLogTty(self) -> None: 

391 """Verify that log output to terminal can be suppressed.""" 

392 with self.runner.isolated_filesystem(): 

393 for log_tty in (True, False): 

394 # The pytest log handler interferes with the log configuration 

395 # settings set up by initLog -- therefore test by using 

396 # a subprocess. 

397 if log_tty: 

398 args = ("butler", "--log-level", "DEBUG", "--log-tty", "create", "here") 

399 else: 

400 args = ("butler", "--log-level", "DEBUG", "--no-log-tty", "create", "here2") 

401 result = subprocess.run(args, capture_output=True) 

402 

403 output = result.stderr.decode() 

404 if log_tty: 

405 self.assertIn("DEBUG", output) 

406 else: 

407 self.assertNotIn("DEBUG", output) 

408 

409 

410if __name__ == "__main__": 

411 unittest.main()