Coverage for python/lsst/daf/butler/tests/cliLogTestBase.py: 32%

136 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:53 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for the daf_butler CliLog utility. 

29 

30Code is implemented in 

31daf_butler but some only runs if lsst.log.Log can be imported so these parts of 

32it can't be tested there because daf_butler does not directly depend on 

33lsst.log, and only uses it if it has been setup by another package. 

34""" 

35 

36from __future__ import annotations 

37 

38import logging 

39import os 

40import re 

41import subprocess 

42import tempfile 

43import unittest 

44from collections import namedtuple 

45from collections.abc import Callable 

46from functools import partial 

47from io import StringIO 

48from logging import DEBUG, INFO, WARNING 

49from typing import TYPE_CHECKING, Any 

50 

51import click 

52from lsst.daf.butler.cli.butler import cli as butlerCli 

53from lsst.daf.butler.cli.cliLog import CliLog 

54from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, command_test_env 

55from lsst.daf.butler.logging import ButlerLogRecords 

56from lsst.utils.logging import TRACE 

57 

58try: 

59 import lsst.log as lsstLog 

60 

61 lsstLog_INFO = lsstLog.INFO 

62 lsstLog_DEBUG = lsstLog.DEBUG 

63 lsstLog_WARN = lsstLog.WARN 

64except ModuleNotFoundError: 

65 lsstLog = None 

66 lsstLog_INFO = 0 

67 lsstLog_DEBUG = 0 

68 lsstLog_WARN = 0 

69 

70 

71@click.command() 

72@click.option("--expected-pyroot-level", type=int) 

73@click.option("--expected-pylsst-level", type=int) 

74@click.option("--expected-pybutler-level", type=int) 

75@click.option("--expected-lsstroot-level", type=int) 

76@click.option("--expected-lsstbutler-level", type=int) 

77@click.option("--expected-lsstx-level", type=int) 

78def command_log_settings_test( 

79 expected_pyroot_level: str, 

80 expected_pylsst_level: str, 

81 expected_pybutler_level: str, 

82 expected_lsstroot_level: str, 

83 expected_lsstbutler_level: str, 

84 expected_lsstx_level: str, 

85) -> None: 

86 """Test command-line log settings.""" 

87 LogLevel = namedtuple("LogLevel", ("expected", "actual", "name")) 

88 

89 logLevels = [ 

90 LogLevel(expected_pyroot_level, logging.getLogger().level, "pyRoot"), 

91 LogLevel(expected_pylsst_level, logging.getLogger("lsst").getEffectiveLevel(), "pyLsst"), 

92 LogLevel( 

93 expected_pybutler_level, logging.getLogger("lsst.daf.butler").getEffectiveLevel(), "pyButler" 

94 ), 

95 LogLevel(expected_lsstx_level, logging.getLogger("lsstx").getEffectiveLevel(), "pyLsstx"), 

96 ] 

97 if lsstLog is not None: 

98 logLevels.extend( 

99 [ 

100 LogLevel(expected_lsstroot_level, lsstLog.getLogger("lsst").getEffectiveLevel(), "lsstRoot"), 

101 LogLevel( 

102 expected_lsstbutler_level, 

103 lsstLog.getLogger("lsst.daf.butler").getEffectiveLevel(), 

104 "lsstButler", 

105 ), 

106 ] 

107 ) 

108 for expected, actual, name in logLevels: 

109 if expected != actual: 

110 raise ( 

111 click.ClickException(message=f"expected {name} level to be {expected!r}, actual:{actual!r}") 

112 ) 

113 

114 

115class CliLogTestBase: 

116 """Tests log initialization, reset, and setting log levels.""" 

117 

118 if TYPE_CHECKING: 

119 assertEqual: Callable 

120 assertIn: Callable 

121 assertTrue: Callable 

122 assertFalse: Callable 

123 assertGreater: Callable 

124 subTest: Callable 

125 assertNotIn: Callable 

126 

127 def setUp(self) -> None: 

128 self.runner = LogCliRunner() 

129 

130 class PythonLogger: 

131 """Keeps track of log level of a component and number of handlers 

132 attached to it at the time this object was initialized. 

133 """ 

134 

135 def __init__(self, component: str | None) -> None: 

136 self.logger = logging.getLogger(component) 

137 self.initialLevel = self.logger.level 

138 

139 class LsstLogger: 

140 """Keeps track of log level for a component at the time this object was 

141 initialized. 

142 """ 

143 

144 def __init__(self, component: str) -> None: 

145 self.logger = lsstLog.getLogger(component) if lsstLog else None 

146 self.initialLevel = self.logger.getLevel() if lsstLog else None 

147 

148 def runTest(self, cmd: Callable) -> None: 

149 """Test that the log context manager works with the butler cli to 

150 initialize the logging system according to cli inputs for the duration 

151 of the command execution and resets the logging system to its previous 

152 state or expected state when command execution finishes. 

153 """ 

154 pyRoot = self.PythonLogger(None) 

155 pyButler = self.PythonLogger("lsst.daf.butler") 

156 pyLsstRoot = self.PythonLogger("lsst") 

157 lsstRoot = self.LsstLogger("") 

158 lsstButler = self.LsstLogger("lsst.daf.butler") 

159 

160 with command_test_env( 

161 self.runner, "lsst.daf.butler.tests.cliLogTestBase", "command-log-settings-test" 

162 ): 

163 result = cmd() 

164 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

165 

166 # The test environment may have changed the python root logger 

167 # so we can not assume it will be WARNING. 

168 self.assertEqual(pyRoot.logger.level, logging.getLogger().getEffectiveLevel()) 

169 self.assertEqual(pyLsstRoot.logger.level, logging.INFO) 

170 self.assertEqual(pyButler.logger.level, pyButler.initialLevel) 

171 if lsstLog is not None: 

172 self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO) 

173 # lsstLogLevel can either be the initial level, or uninitialized or 

174 # the defined default value. 

175 expectedLsstLogLevel = ( 

176 (lsstButler.initialLevel,) 

177 if lsstButler.initialLevel != -1 

178 else (-1, CliLog.defaultLsstLogLevel) 

179 ) 

180 self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel) 

181 

182 def test_butlerCliLog(self) -> None: 

183 """Test that the log context manager works with the butler cli to 

184 initialize the logging system according to cli inputs for the duration 

185 of the command execution and resets the logging system to its previous 

186 state or expected state when command execution finishes. 

187 """ 

188 # Run with two different log level settings. 

189 log_levels = ( 

190 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

191 # lsstroot, lsstbutler, lsstx 

192 ( 

193 "WARNING", 

194 "lsst.daf.butler=DEBUG", 

195 WARNING, 

196 WARNING, 

197 DEBUG, 

198 lsstLog_WARN, 

199 lsstLog_DEBUG, 

200 WARNING, 

201 ), 

202 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, WARNING), 

203 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, DEBUG), 

204 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

205 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, DEBUG), 

206 ) 

207 

208 self._test_levels(log_levels) 

209 

210 # Check that the environment variable can set additional roots. 

211 log_levels = ( 

212 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

213 # lsstroot, lsstbutler, lsstx 

214 ( 

215 "WARNING", 

216 "lsst.daf.butler=DEBUG", 

217 WARNING, 

218 WARNING, 

219 DEBUG, 

220 lsstLog_WARN, 

221 lsstLog_DEBUG, 

222 WARNING, 

223 ), 

224 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

225 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, INFO), 

226 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

227 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, INFO), 

228 ) 

229 

230 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_ROOT_LOGGER": "lsstx"}): 

231 self._test_levels(log_levels) 

232 

233 def _test_levels(self, log_levels: tuple[tuple[str, str, int, int, int, Any, Any, int], ...]) -> None: 

234 for level1, level2, x_pyroot, x_pylsst, x_pybutler, x_lsstroot, x_lsstbutler, x_lsstx in log_levels: 

235 with self.subTest("Test different log levels", level1=level1, level2=level2): 

236 self.runTest( 

237 partial( 

238 self.runner.invoke, 

239 butlerCli, 

240 [ 

241 "--log-level", 

242 level1, 

243 "--log-level", 

244 level2, 

245 "command-log-settings-test", 

246 "--expected-pyroot-level", 

247 x_pyroot, 

248 "--expected-pylsst-level", 

249 x_pylsst, 

250 "--expected-pybutler-level", 

251 x_pybutler, 

252 "--expected-lsstroot-level", 

253 x_lsstroot, 

254 "--expected-lsstbutler-level", 

255 x_lsstbutler, 

256 "--expected-lsstx-level", 

257 x_lsstx, 

258 ], 

259 ) 

260 ) 

261 

262 def test_helpLogReset(self) -> None: 

263 """Verify that when a command does not execute, like when the help menu 

264 is printed instead, that CliLog is still reset. 

265 """ 

266 self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"])) 

267 

268 def testLongLog(self) -> None: 

269 """Verify the timestamp is in the log messages when the --long-log 

270 flag is set. 

271 """ 

272 # When longlog=True, loglines start with the log level and a 

273 # timestamp with the following format: 

274 # "year-month-day T hour-minute-second.millisecond-zoneoffset" 

275 # For example: "DEBUG 2020-10-28T10:20:31-07:00 ..."" 

276 # The log level name can change, we verify there is an all 

277 # caps word there but do not verify the word. We do not verify 

278 # the rest of the log string, assume that if the timestamp is 

279 # in the string that the rest of the string will appear as 

280 # expected. 

281 timestampRegex = re.compile( 

282 r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})" 

283 "?([-,+][01][0-9]:[034][05]|Z) .*" 

284 ) 

285 

286 # When longlog=False, log lines start with the module name and 

287 # log level, for example: 

288 # lsst.daf.butler.config DEBUG: ... 

289 modulesRegex = re.compile(r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*") 

290 

291 with self.runner.isolated_filesystem(): 

292 for longlog in (True, False): 

293 # The pytest log handler interferes with the log configuration 

294 # settings set up by initLog -- therefore test by using 

295 # a subprocess. 

296 args: tuple[str, ...] 

297 if longlog: 

298 args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here") 

299 else: 

300 args = ("butler", "--log-level", "DEBUG", "create", "here") 

301 result = subprocess.run(args, capture_output=True) 

302 # There are cases where the newlines are stripped from the log 

303 # output (like in Jenkins), since we can't depend on newlines 

304 # in log output they are removed here from test output. 

305 output = StringIO(result.stderr.decode().replace("\n", " ")) 

306 startedWithTimestamp = any(timestampRegex.match(line) for line in output.readlines()) 

307 output.seek(0) 

308 startedWithModule = any(modulesRegex.match(line) for line in output.readlines()) 

309 if longlog: 

310 self.assertTrue( 

311 startedWithTimestamp, msg=f"did not find timestamp in: \n{output.getvalue()}" 

312 ) 

313 self.assertFalse( 

314 startedWithModule, msg=f"found lines starting with module in: \n{output.getvalue()}" 

315 ) 

316 else: 

317 self.assertFalse(startedWithTimestamp, msg=f"found timestamp in: \n{output.getvalue()}") 

318 self.assertTrue( 

319 startedWithModule, 

320 msg=f"did not find lines starting with module in: \n{output.getvalue()}", 

321 ) 

322 

323 def testFileLogging(self) -> None: 

324 """Test --log-file option.""" 

325 with self.runner.isolated_filesystem(): 

326 for i, suffix in enumerate([".json", ".log"]): 

327 # Get a temporary file name and immediately close it 

328 fd = tempfile.NamedTemporaryFile(suffix=suffix) 

329 filename = fd.name 

330 fd.close() 

331 

332 args = ( 

333 "--log-level", 

334 "DEBUG", 

335 "--log-file", 

336 filename, 

337 "--log-label", 

338 "k1=v1,k2=v2", 

339 "--log-label", 

340 "k3=v3", 

341 "create", 

342 f"here{i}", 

343 ) 

344 

345 result = self.runner.invoke(butlerCli, args) 

346 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

347 

348 # Record to test. Test one in the middle that we know is 

349 # a DEBUG message. The first message might come from 

350 # python itself since warnings are redirected to log 

351 # messages. 

352 num = 4 

353 

354 n_records = 5 

355 if suffix == ".json": 

356 records = ButlerLogRecords.from_file(filename) 

357 self.assertGreater(len(records), num) 

358 self.assertEqual(records[num].levelname, "DEBUG", str(records[num])) 

359 self.assertEqual(records[0].MDC, dict(K1="v1", K2="v2", K3="v3")) 

360 

361 self.assertGreater(len(records), n_records) 

362 else: 

363 with open(filename) as filed: 

364 records_text = filed.readlines() 

365 self.assertGreater(len(records_text), num) 

366 self.assertIn("DEBUG", records_text[num], str(records_text[num])) 

367 self.assertNotIn("{", records_text[num], str(records_text[num])) 

368 

369 self.assertGreater(len(records), n_records) 

370 

371 def testLogTty(self) -> None: 

372 """Verify that log output to terminal can be suppressed.""" 

373 with self.runner.isolated_filesystem(): 

374 for log_tty in (True, False): 

375 # The pytest log handler interferes with the log configuration 

376 # settings set up by initLog -- therefore test by using 

377 # a subprocess. 

378 if log_tty: 

379 args = ("butler", "--log-level", "DEBUG", "--log-tty", "create", "here") 

380 else: 

381 args = ("butler", "--log-level", "DEBUG", "--no-log-tty", "create", "here2") 

382 result = subprocess.run(args, capture_output=True) 

383 

384 output = result.stderr.decode() 

385 if log_tty: 

386 self.assertIn("DEBUG", output) 

387 else: 

388 self.assertNotIn("DEBUG", output) 

389 

390 

391if __name__ == "__main__": 

392 unittest.main()