Coverage for python/lsst/daf/butler/tests/cliLogTestBase.py: 26%

149 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-25 02:36 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23"""Unit tests for the daf_butler CliLog utility. 

24 

25Code is implemented in 

26daf_butler but some only runs if lsst.log.Log can be imported so these parts of 

27it can't be tested there because daf_butler does not directly depend on 

28lsst.log, and only uses it if it has been setup by another package.""" 

29 

30import logging 

31import os 

32import re 

33import subprocess 

34import tempfile 

35import unittest 

36from collections import namedtuple 

37from collections.abc import Callable 

38from functools import partial 

39from io import StringIO 

40from logging import DEBUG, INFO, WARNING 

41from typing import TYPE_CHECKING, Any 

42 

43import click 

44from lsst.daf.butler.cli.butler import cli as butlerCli 

45from lsst.daf.butler.cli.cliLog import CliLog 

46from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, command_test_env 

47from lsst.daf.butler.core.logging import ButlerLogRecords 

48from lsst.utils.logging import TRACE 

49 

50try: 

51 import lsst.log as lsstLog 

52 

53 lsstLog_INFO = lsstLog.INFO 

54 lsstLog_DEBUG = lsstLog.DEBUG 

55 lsstLog_WARN = lsstLog.WARN 

56except ModuleNotFoundError: 

57 lsstLog = None 

58 lsstLog_INFO = 0 

59 lsstLog_DEBUG = 0 

60 lsstLog_WARN = 0 

61 

62 

63@click.command() 

64@click.option("--expected-pyroot-level", type=int) 

65@click.option("--expected-pylsst-level", type=int) 

66@click.option("--expected-pybutler-level", type=int) 

67@click.option("--expected-lsstroot-level", type=int) 

68@click.option("--expected-lsstbutler-level", type=int) 

69@click.option("--expected-lsstx-level", type=int) 

70def command_log_settings_test( 

71 expected_pyroot_level: str, 

72 expected_pylsst_level: str, 

73 expected_pybutler_level: str, 

74 expected_lsstroot_level: str, 

75 expected_lsstbutler_level: str, 

76 expected_lsstx_level: str, 

77) -> None: 

78 

79 LogLevel = namedtuple("LogLevel", ("expected", "actual", "name")) 

80 

81 logLevels = [ 

82 LogLevel(expected_pyroot_level, logging.getLogger().level, "pyRoot"), 

83 LogLevel(expected_pylsst_level, logging.getLogger("lsst").getEffectiveLevel(), "pyLsst"), 

84 LogLevel( 

85 expected_pybutler_level, logging.getLogger("lsst.daf.butler").getEffectiveLevel(), "pyButler" 

86 ), 

87 LogLevel(expected_lsstx_level, logging.getLogger("lsstx").getEffectiveLevel(), "pyLsstx"), 

88 ] 

89 if lsstLog is not None: 

90 logLevels.extend( 

91 [ 

92 LogLevel(expected_lsstroot_level, lsstLog.getLogger("lsst").getEffectiveLevel(), "lsstRoot"), 

93 LogLevel( 

94 expected_lsstbutler_level, 

95 lsstLog.getLogger("lsst.daf.butler").getEffectiveLevel(), 

96 "lsstButler", 

97 ), 

98 ] 

99 ) 

100 for expected, actual, name in logLevels: 

101 if expected != actual: 

102 raise ( 

103 click.ClickException(message=f"expected {name} level to be {expected!r}, actual:{actual!r}") 

104 ) 

105 

106 

107class CliLogTestBase: 

108 """Tests log initialization, reset, and setting log levels.""" 

109 

110 if TYPE_CHECKING: 110 ↛ 111line 110 didn't jump to line 111, because the condition on line 110 was never true

111 assertEqual: Callable 

112 assertIn: Callable 

113 assertTrue: Callable 

114 assertFalse: Callable 

115 assertGreater: Callable 

116 subTest: Callable 

117 assertNotIn: Callable 

118 

119 lsstLogHandlerId = None 

120 

121 def setUp(self) -> None: 

122 self.runner = LogCliRunner() 

123 

124 def tearDown(self) -> None: 

125 self.lsstLogHandlerId = None 

126 

127 class PythonLogger: 

128 """Keeps track of log level of a component and number of handlers 

129 attached to it at the time this object was initialized.""" 

130 

131 def __init__(self, component: str | None) -> None: 

132 self.logger = logging.getLogger(component) 

133 self.initialLevel = self.logger.level 

134 

135 class LsstLogger: 

136 """Keeps track of log level for a component at the time this object was 

137 initialized.""" 

138 

139 def __init__(self, component: str) -> None: 

140 self.logger = lsstLog.getLogger(component) if lsstLog else None 

141 self.initialLevel = self.logger.getLevel() if lsstLog else None 

142 

143 def runTest(self, cmd: Callable) -> None: 

144 """Test that the log context manager works with the butler cli to 

145 initialize the logging system according to cli inputs for the duration 

146 of the command execution and resets the logging system to its previous 

147 state or expected state when command execution finishes.""" 

148 pyRoot = self.PythonLogger(None) 

149 pyButler = self.PythonLogger("lsst.daf.butler") 

150 pyLsstRoot = self.PythonLogger("lsst") 

151 lsstRoot = self.LsstLogger("") 

152 lsstButler = self.LsstLogger("lsst.daf.butler") 

153 

154 with command_test_env( 

155 self.runner, "lsst.daf.butler.tests.cliLogTestBase", "command-log-settings-test" 

156 ): 

157 result = cmd() 

158 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

159 

160 self.assertEqual(pyRoot.logger.level, logging.WARNING) 

161 self.assertEqual(pyLsstRoot.logger.level, logging.INFO) 

162 self.assertEqual(pyButler.logger.level, pyButler.initialLevel) 

163 if lsstLog is not None: 

164 self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO) 

165 # lsstLogLevel can either be the initial level, or uninitialized or 

166 # the defined default value. 

167 expectedLsstLogLevel = ( 

168 (lsstButler.initialLevel,) 

169 if lsstButler.initialLevel != -1 

170 else (-1, CliLog.defaultLsstLogLevel) 

171 ) 

172 self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel) 

173 

174 def test_butlerCliLog(self) -> None: 

175 """Test that the log context manager works with the butler cli to 

176 initialize the logging system according to cli inputs for the duration 

177 of the command execution and resets the logging system to its previous 

178 state or expected state when command execution finishes.""" 

179 

180 # Run with two different log level settings. 

181 log_levels = ( 

182 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

183 # lsstroot, lsstbutler, lsstx 

184 ( 

185 "WARNING", 

186 "lsst.daf.butler=DEBUG", 

187 WARNING, 

188 WARNING, 

189 DEBUG, 

190 lsstLog_WARN, 

191 lsstLog_DEBUG, 

192 WARNING, 

193 ), 

194 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, WARNING), 

195 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, DEBUG), 

196 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

197 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, DEBUG), 

198 ) 

199 

200 self._test_levels(log_levels) 

201 

202 # Check that the environment variable can set additional roots. 

203 log_levels = ( 

204 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

205 # lsstroot, lsstbutler, lsstx 

206 ( 

207 "WARNING", 

208 "lsst.daf.butler=DEBUG", 

209 WARNING, 

210 WARNING, 

211 DEBUG, 

212 lsstLog_WARN, 

213 lsstLog_DEBUG, 

214 WARNING, 

215 ), 

216 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

217 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, INFO), 

218 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

219 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, INFO), 

220 ) 

221 

222 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_ROOT_LOGGER": "lsstx"}): 

223 self._test_levels(log_levels) 

224 

225 def _test_levels(self, log_levels: tuple[tuple[str, str, int, int, int, Any, Any, int], ...]) -> None: 

226 for level1, level2, x_pyroot, x_pylsst, x_pybutler, x_lsstroot, x_lsstbutler, x_lsstx in log_levels: 

227 with self.subTest("Test different log levels", level1=level1, level2=level2): 

228 self.runTest( 

229 partial( 

230 self.runner.invoke, 

231 butlerCli, 

232 [ 

233 "--log-level", 

234 level1, 

235 "--log-level", 

236 level2, 

237 "command-log-settings-test", 

238 "--expected-pyroot-level", 

239 x_pyroot, 

240 "--expected-pylsst-level", 

241 x_pylsst, 

242 "--expected-pybutler-level", 

243 x_pybutler, 

244 "--expected-lsstroot-level", 

245 x_lsstroot, 

246 "--expected-lsstbutler-level", 

247 x_lsstbutler, 

248 "--expected-lsstx-level", 

249 x_lsstx, 

250 ], 

251 ) 

252 ) 

253 

254 def test_helpLogReset(self) -> None: 

255 """Verify that when a command does not execute, like when the help menu 

256 is printed instead, that CliLog is still reset.""" 

257 

258 self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"])) 

259 

260 def testLongLog(self) -> None: 

261 """Verify the timestamp is in the log messages when the --long-log 

262 flag is set.""" 

263 

264 # When longlog=True, loglines start with the log level and a 

265 # timestamp with the following format: 

266 # "year-month-day T hour-minute-second.millisecond-zoneoffset" 

267 # For example: "DEBUG 2020-10-28T10:20:31-07:00 ..."" 

268 # The log level name can change, we verify there is an all 

269 # caps word there but do not verify the word. We do not verify 

270 # the rest of the log string, assume that if the timestamp is 

271 # in the string that the rest of the string will appear as 

272 # expected. 

273 timestampRegex = re.compile( 

274 r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})" 

275 "?([-,+][01][0-9]:[034][05]|Z) .*" 

276 ) 

277 

278 # When longlog=False, log lines start with the module name and 

279 # log level, for example: 

280 # lsst.daf.butler.core.config DEBUG: ... 

281 modulesRegex = re.compile(r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*") 

282 

283 with self.runner.isolated_filesystem(): 

284 for longlog in (True, False): 

285 # The pytest log handler interferes with the log configuration 

286 # settings set up by initLog -- therefore test by using 

287 # a subprocess. 

288 args: tuple[str, ...] 

289 if longlog: 

290 args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here") 

291 else: 

292 args = ("butler", "--log-level", "DEBUG", "create", "here") 

293 result = subprocess.run(args, capture_output=True) 

294 # There are cases where the newlines are stripped from the log 

295 # output (like in Jenkins), since we can't depend on newlines 

296 # in log output they are removed here from test output. 

297 output = StringIO((result.stderr.decode().replace("\n", " "))) 

298 startedWithTimestamp = any([timestampRegex.match(line) for line in output.readlines()]) 

299 output.seek(0) 

300 startedWithModule = any(modulesRegex.match(line) for line in output.readlines()) 

301 if longlog: 

302 self.assertTrue( 

303 startedWithTimestamp, msg=f"did not find timestamp in: \n{output.getvalue()}" 

304 ) 

305 self.assertFalse( 

306 startedWithModule, msg=f"found lines starting with module in: \n{output.getvalue()}" 

307 ) 

308 else: 

309 self.assertFalse(startedWithTimestamp, msg=f"found timestamp in: \n{output.getvalue()}") 

310 self.assertTrue( 

311 startedWithModule, 

312 msg=f"did not find lines starting with module in: \n{output.getvalue()}", 

313 ) 

314 

315 def testFileLogging(self) -> None: 

316 """Test --log-file option.""" 

317 with self.runner.isolated_filesystem(): 

318 for i, suffix in enumerate([".json", ".log"]): 

319 # Get a temporary file name and immediately close it 

320 fd = tempfile.NamedTemporaryFile(suffix=suffix) 

321 filename = fd.name 

322 fd.close() 

323 

324 args = ( 

325 "--log-level", 

326 "DEBUG", 

327 "--log-file", 

328 filename, 

329 "--log-label", 

330 "k1=v1,k2=v2", 

331 "--log-label", 

332 "k3=v3", 

333 "create", 

334 f"here{i}", 

335 ) 

336 

337 result = self.runner.invoke(butlerCli, args) 

338 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

339 

340 # Record to test. Test one in the middle that we know is 

341 # a DEBUG message. The first message might come from 

342 # python itself since warnings are redirected to log 

343 # messages. 

344 num = 4 

345 

346 n_records = 5 

347 if suffix == ".json": 

348 records = ButlerLogRecords.from_file(filename) 

349 self.assertGreater(len(records), num) 

350 self.assertEqual(records[num].levelname, "DEBUG", str(records[num])) 

351 self.assertEqual(records[0].MDC, dict(K1="v1", K2="v2", K3="v3")) 

352 

353 self.assertGreater(len(records), n_records) 

354 else: 

355 with open(filename) as filed: 

356 records_text = filed.readlines() 

357 self.assertGreater(len(records_text), num) 

358 self.assertIn("DEBUG", records_text[num], str(records_text[num])) 

359 self.assertNotIn("{", records_text[num], str(records_text[num])) 

360 

361 self.assertGreater(len(records), n_records) 

362 

363 def testLogTty(self) -> None: 

364 """Verify that log output to terminal can be suppressed.""" 

365 

366 with self.runner.isolated_filesystem(): 

367 for log_tty in (True, False): 

368 # The pytest log handler interferes with the log configuration 

369 # settings set up by initLog -- therefore test by using 

370 # a subprocess. 

371 if log_tty: 

372 args = ("butler", "--log-level", "DEBUG", "--log-tty", "create", "here") 

373 else: 

374 args = ("butler", "--log-level", "DEBUG", "--no-log-tty", "create", "here2") 

375 result = subprocess.run(args, capture_output=True) 

376 

377 output = result.stderr.decode() 

378 if log_tty: 

379 self.assertIn("DEBUG", output) 

380 else: 

381 self.assertNotIn("DEBUG", output) 

382 

383 

384if __name__ == "__main__": 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true

385 unittest.main()