Coverage for python/lsst/daf/butler/tests/cliLogTestBase.py: 26%

149 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-08 10:28 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23"""Unit tests for the daf_butler CliLog utility. 

24 

25Code is implemented in 

26daf_butler but some only runs if lsst.log.Log can be imported so these parts of 

27it can't be tested there because daf_butler does not directly depend on 

28lsst.log, and only uses it if it has been setup by another package.""" 

29 

30import logging 

31import os 

32import re 

33import subprocess 

34import tempfile 

35import unittest 

36from collections import namedtuple 

37from collections.abc import Callable 

38from functools import partial 

39from io import StringIO 

40from logging import DEBUG, INFO, WARNING 

41from typing import TYPE_CHECKING, Any 

42 

43import click 

44from lsst.daf.butler.cli.butler import cli as butlerCli 

45from lsst.daf.butler.cli.cliLog import CliLog 

46from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, command_test_env 

47from lsst.daf.butler.core.logging import ButlerLogRecords 

48from lsst.utils.logging import TRACE 

49 

50try: 

51 import lsst.log as lsstLog 

52 

53 lsstLog_INFO = lsstLog.INFO 

54 lsstLog_DEBUG = lsstLog.DEBUG 

55 lsstLog_WARN = lsstLog.WARN 

56except ModuleNotFoundError: 

57 lsstLog = None 

58 lsstLog_INFO = 0 

59 lsstLog_DEBUG = 0 

60 lsstLog_WARN = 0 

61 

62 

63@click.command() 

64@click.option("--expected-pyroot-level", type=int) 

65@click.option("--expected-pylsst-level", type=int) 

66@click.option("--expected-pybutler-level", type=int) 

67@click.option("--expected-lsstroot-level", type=int) 

68@click.option("--expected-lsstbutler-level", type=int) 

69@click.option("--expected-lsstx-level", type=int) 

70def command_log_settings_test( 

71 expected_pyroot_level: str, 

72 expected_pylsst_level: str, 

73 expected_pybutler_level: str, 

74 expected_lsstroot_level: str, 

75 expected_lsstbutler_level: str, 

76 expected_lsstx_level: str, 

77) -> None: 

78 LogLevel = namedtuple("LogLevel", ("expected", "actual", "name")) 

79 

80 logLevels = [ 

81 LogLevel(expected_pyroot_level, logging.getLogger().level, "pyRoot"), 

82 LogLevel(expected_pylsst_level, logging.getLogger("lsst").getEffectiveLevel(), "pyLsst"), 

83 LogLevel( 

84 expected_pybutler_level, logging.getLogger("lsst.daf.butler").getEffectiveLevel(), "pyButler" 

85 ), 

86 LogLevel(expected_lsstx_level, logging.getLogger("lsstx").getEffectiveLevel(), "pyLsstx"), 

87 ] 

88 if lsstLog is not None: 

89 logLevels.extend( 

90 [ 

91 LogLevel(expected_lsstroot_level, lsstLog.getLogger("lsst").getEffectiveLevel(), "lsstRoot"), 

92 LogLevel( 

93 expected_lsstbutler_level, 

94 lsstLog.getLogger("lsst.daf.butler").getEffectiveLevel(), 

95 "lsstButler", 

96 ), 

97 ] 

98 ) 

99 for expected, actual, name in logLevels: 

100 if expected != actual: 

101 raise ( 

102 click.ClickException(message=f"expected {name} level to be {expected!r}, actual:{actual!r}") 

103 ) 

104 

105 

106class CliLogTestBase: 

107 """Tests log initialization, reset, and setting log levels.""" 

108 

109 if TYPE_CHECKING: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true

110 assertEqual: Callable 

111 assertIn: Callable 

112 assertTrue: Callable 

113 assertFalse: Callable 

114 assertGreater: Callable 

115 subTest: Callable 

116 assertNotIn: Callable 

117 

118 lsstLogHandlerId = None 

119 

120 def setUp(self) -> None: 

121 self.runner = LogCliRunner() 

122 

123 def tearDown(self) -> None: 

124 self.lsstLogHandlerId = None 

125 

126 class PythonLogger: 

127 """Keeps track of log level of a component and number of handlers 

128 attached to it at the time this object was initialized.""" 

129 

130 def __init__(self, component: str | None) -> None: 

131 self.logger = logging.getLogger(component) 

132 self.initialLevel = self.logger.level 

133 

134 class LsstLogger: 

135 """Keeps track of log level for a component at the time this object was 

136 initialized.""" 

137 

138 def __init__(self, component: str) -> None: 

139 self.logger = lsstLog.getLogger(component) if lsstLog else None 

140 self.initialLevel = self.logger.getLevel() if lsstLog else None 

141 

142 def runTest(self, cmd: Callable) -> None: 

143 """Test that the log context manager works with the butler cli to 

144 initialize the logging system according to cli inputs for the duration 

145 of the command execution and resets the logging system to its previous 

146 state or expected state when command execution finishes.""" 

147 pyRoot = self.PythonLogger(None) 

148 pyButler = self.PythonLogger("lsst.daf.butler") 

149 pyLsstRoot = self.PythonLogger("lsst") 

150 lsstRoot = self.LsstLogger("") 

151 lsstButler = self.LsstLogger("lsst.daf.butler") 

152 

153 with command_test_env( 

154 self.runner, "lsst.daf.butler.tests.cliLogTestBase", "command-log-settings-test" 

155 ): 

156 result = cmd() 

157 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

158 

159 self.assertEqual(pyRoot.logger.level, logging.WARNING) 

160 self.assertEqual(pyLsstRoot.logger.level, logging.INFO) 

161 self.assertEqual(pyButler.logger.level, pyButler.initialLevel) 

162 if lsstLog is not None: 

163 self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO) 

164 # lsstLogLevel can either be the initial level, or uninitialized or 

165 # the defined default value. 

166 expectedLsstLogLevel = ( 

167 (lsstButler.initialLevel,) 

168 if lsstButler.initialLevel != -1 

169 else (-1, CliLog.defaultLsstLogLevel) 

170 ) 

171 self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel) 

172 

173 def test_butlerCliLog(self) -> None: 

174 """Test that the log context manager works with the butler cli to 

175 initialize the logging system according to cli inputs for the duration 

176 of the command execution and resets the logging system to its previous 

177 state or expected state when command execution finishes.""" 

178 

179 # Run with two different log level settings. 

180 log_levels = ( 

181 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

182 # lsstroot, lsstbutler, lsstx 

183 ( 

184 "WARNING", 

185 "lsst.daf.butler=DEBUG", 

186 WARNING, 

187 WARNING, 

188 DEBUG, 

189 lsstLog_WARN, 

190 lsstLog_DEBUG, 

191 WARNING, 

192 ), 

193 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, WARNING), 

194 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, DEBUG), 

195 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

196 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, DEBUG), 

197 ) 

198 

199 self._test_levels(log_levels) 

200 

201 # Check that the environment variable can set additional roots. 

202 log_levels = ( 

203 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

204 # lsstroot, lsstbutler, lsstx 

205 ( 

206 "WARNING", 

207 "lsst.daf.butler=DEBUG", 

208 WARNING, 

209 WARNING, 

210 DEBUG, 

211 lsstLog_WARN, 

212 lsstLog_DEBUG, 

213 WARNING, 

214 ), 

215 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

216 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, INFO), 

217 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

218 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, INFO), 

219 ) 

220 

221 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_ROOT_LOGGER": "lsstx"}): 

222 self._test_levels(log_levels) 

223 

224 def _test_levels(self, log_levels: tuple[tuple[str, str, int, int, int, Any, Any, int], ...]) -> None: 

225 for level1, level2, x_pyroot, x_pylsst, x_pybutler, x_lsstroot, x_lsstbutler, x_lsstx in log_levels: 

226 with self.subTest("Test different log levels", level1=level1, level2=level2): 

227 self.runTest( 

228 partial( 

229 self.runner.invoke, 

230 butlerCli, 

231 [ 

232 "--log-level", 

233 level1, 

234 "--log-level", 

235 level2, 

236 "command-log-settings-test", 

237 "--expected-pyroot-level", 

238 x_pyroot, 

239 "--expected-pylsst-level", 

240 x_pylsst, 

241 "--expected-pybutler-level", 

242 x_pybutler, 

243 "--expected-lsstroot-level", 

244 x_lsstroot, 

245 "--expected-lsstbutler-level", 

246 x_lsstbutler, 

247 "--expected-lsstx-level", 

248 x_lsstx, 

249 ], 

250 ) 

251 ) 

252 

253 def test_helpLogReset(self) -> None: 

254 """Verify that when a command does not execute, like when the help menu 

255 is printed instead, that CliLog is still reset.""" 

256 

257 self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"])) 

258 

259 def testLongLog(self) -> None: 

260 """Verify the timestamp is in the log messages when the --long-log 

261 flag is set.""" 

262 

263 # When longlog=True, loglines start with the log level and a 

264 # timestamp with the following format: 

265 # "year-month-day T hour-minute-second.millisecond-zoneoffset" 

266 # For example: "DEBUG 2020-10-28T10:20:31-07:00 ..."" 

267 # The log level name can change, we verify there is an all 

268 # caps word there but do not verify the word. We do not verify 

269 # the rest of the log string, assume that if the timestamp is 

270 # in the string that the rest of the string will appear as 

271 # expected. 

272 timestampRegex = re.compile( 

273 r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})" 

274 "?([-,+][01][0-9]:[034][05]|Z) .*" 

275 ) 

276 

277 # When longlog=False, log lines start with the module name and 

278 # log level, for example: 

279 # lsst.daf.butler.core.config DEBUG: ... 

280 modulesRegex = re.compile(r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*") 

281 

282 with self.runner.isolated_filesystem(): 

283 for longlog in (True, False): 

284 # The pytest log handler interferes with the log configuration 

285 # settings set up by initLog -- therefore test by using 

286 # a subprocess. 

287 args: tuple[str, ...] 

288 if longlog: 

289 args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here") 

290 else: 

291 args = ("butler", "--log-level", "DEBUG", "create", "here") 

292 result = subprocess.run(args, capture_output=True) 

293 # There are cases where the newlines are stripped from the log 

294 # output (like in Jenkins), since we can't depend on newlines 

295 # in log output they are removed here from test output. 

296 output = StringIO((result.stderr.decode().replace("\n", " "))) 

297 startedWithTimestamp = any([timestampRegex.match(line) for line in output.readlines()]) 

298 output.seek(0) 

299 startedWithModule = any(modulesRegex.match(line) for line in output.readlines()) 

300 if longlog: 

301 self.assertTrue( 

302 startedWithTimestamp, msg=f"did not find timestamp in: \n{output.getvalue()}" 

303 ) 

304 self.assertFalse( 

305 startedWithModule, msg=f"found lines starting with module in: \n{output.getvalue()}" 

306 ) 

307 else: 

308 self.assertFalse(startedWithTimestamp, msg=f"found timestamp in: \n{output.getvalue()}") 

309 self.assertTrue( 

310 startedWithModule, 

311 msg=f"did not find lines starting with module in: \n{output.getvalue()}", 

312 ) 

313 

314 def testFileLogging(self) -> None: 

315 """Test --log-file option.""" 

316 with self.runner.isolated_filesystem(): 

317 for i, suffix in enumerate([".json", ".log"]): 

318 # Get a temporary file name and immediately close it 

319 fd = tempfile.NamedTemporaryFile(suffix=suffix) 

320 filename = fd.name 

321 fd.close() 

322 

323 args = ( 

324 "--log-level", 

325 "DEBUG", 

326 "--log-file", 

327 filename, 

328 "--log-label", 

329 "k1=v1,k2=v2", 

330 "--log-label", 

331 "k3=v3", 

332 "create", 

333 f"here{i}", 

334 ) 

335 

336 result = self.runner.invoke(butlerCli, args) 

337 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

338 

339 # Record to test. Test one in the middle that we know is 

340 # a DEBUG message. The first message might come from 

341 # python itself since warnings are redirected to log 

342 # messages. 

343 num = 4 

344 

345 n_records = 5 

346 if suffix == ".json": 

347 records = ButlerLogRecords.from_file(filename) 

348 self.assertGreater(len(records), num) 

349 self.assertEqual(records[num].levelname, "DEBUG", str(records[num])) 

350 self.assertEqual(records[0].MDC, dict(K1="v1", K2="v2", K3="v3")) 

351 

352 self.assertGreater(len(records), n_records) 

353 else: 

354 with open(filename) as filed: 

355 records_text = filed.readlines() 

356 self.assertGreater(len(records_text), num) 

357 self.assertIn("DEBUG", records_text[num], str(records_text[num])) 

358 self.assertNotIn("{", records_text[num], str(records_text[num])) 

359 

360 self.assertGreater(len(records), n_records) 

361 

362 def testLogTty(self) -> None: 

363 """Verify that log output to terminal can be suppressed.""" 

364 

365 with self.runner.isolated_filesystem(): 

366 for log_tty in (True, False): 

367 # The pytest log handler interferes with the log configuration 

368 # settings set up by initLog -- therefore test by using 

369 # a subprocess. 

370 if log_tty: 

371 args = ("butler", "--log-level", "DEBUG", "--log-tty", "create", "here") 

372 else: 

373 args = ("butler", "--log-level", "DEBUG", "--no-log-tty", "create", "here2") 

374 result = subprocess.run(args, capture_output=True) 

375 

376 output = result.stderr.decode() 

377 if log_tty: 

378 self.assertIn("DEBUG", output) 

379 else: 

380 self.assertNotIn("DEBUG", output) 

381 

382 

383if __name__ == "__main__": 383 ↛ 384line 383 didn't jump to line 384, because the condition on line 383 was never true

384 unittest.main()