Coverage for python/lsst/daf/butler/tests/cliLogTestBase.py: 26%

136 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 10:56 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for the daf_butler CliLog utility. 

23 

24Code is implemented in 

25daf_butler but some only runs if lsst.log.Log can be imported so these parts of 

26it can't be tested there because daf_butler does not directly depend on 

27lsst.log, and only uses it if it has been setup by another package. 

28""" 

29 

30from __future__ import annotations 

31 

32import logging 

33import os 

34import re 

35import subprocess 

36import tempfile 

37import unittest 

38from collections import namedtuple 

39from collections.abc import Callable 

40from functools import partial 

41from io import StringIO 

42from logging import DEBUG, INFO, WARNING 

43from typing import TYPE_CHECKING, Any 

44 

45import click 

46from lsst.daf.butler.cli.butler import cli as butlerCli 

47from lsst.daf.butler.cli.cliLog import CliLog 

48from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, command_test_env 

49from lsst.daf.butler.core.logging import ButlerLogRecords 

50from lsst.utils.logging import TRACE 

51 

52try: 

53 import lsst.log as lsstLog 

54 

55 lsstLog_INFO = lsstLog.INFO 

56 lsstLog_DEBUG = lsstLog.DEBUG 

57 lsstLog_WARN = lsstLog.WARN 

58except ModuleNotFoundError: 

59 lsstLog = None 

60 lsstLog_INFO = 0 

61 lsstLog_DEBUG = 0 

62 lsstLog_WARN = 0 

63 

64 

65@click.command() 

66@click.option("--expected-pyroot-level", type=int) 

67@click.option("--expected-pylsst-level", type=int) 

68@click.option("--expected-pybutler-level", type=int) 

69@click.option("--expected-lsstroot-level", type=int) 

70@click.option("--expected-lsstbutler-level", type=int) 

71@click.option("--expected-lsstx-level", type=int) 

72def command_log_settings_test( 

73 expected_pyroot_level: str, 

74 expected_pylsst_level: str, 

75 expected_pybutler_level: str, 

76 expected_lsstroot_level: str, 

77 expected_lsstbutler_level: str, 

78 expected_lsstx_level: str, 

79) -> None: 

80 """Test command-line log settings.""" 

81 LogLevel = namedtuple("LogLevel", ("expected", "actual", "name")) 

82 

83 logLevels = [ 

84 LogLevel(expected_pyroot_level, logging.getLogger().level, "pyRoot"), 

85 LogLevel(expected_pylsst_level, logging.getLogger("lsst").getEffectiveLevel(), "pyLsst"), 

86 LogLevel( 

87 expected_pybutler_level, logging.getLogger("lsst.daf.butler").getEffectiveLevel(), "pyButler" 

88 ), 

89 LogLevel(expected_lsstx_level, logging.getLogger("lsstx").getEffectiveLevel(), "pyLsstx"), 

90 ] 

91 if lsstLog is not None: 

92 logLevels.extend( 

93 [ 

94 LogLevel(expected_lsstroot_level, lsstLog.getLogger("lsst").getEffectiveLevel(), "lsstRoot"), 

95 LogLevel( 

96 expected_lsstbutler_level, 

97 lsstLog.getLogger("lsst.daf.butler").getEffectiveLevel(), 

98 "lsstButler", 

99 ), 

100 ] 

101 ) 

102 for expected, actual, name in logLevels: 

103 if expected != actual: 

104 raise ( 

105 click.ClickException(message=f"expected {name} level to be {expected!r}, actual:{actual!r}") 

106 ) 

107 

108 

109class CliLogTestBase: 

110 """Tests log initialization, reset, and setting log levels.""" 

111 

112 if TYPE_CHECKING: 

113 assertEqual: Callable 

114 assertIn: Callable 

115 assertTrue: Callable 

116 assertFalse: Callable 

117 assertGreater: Callable 

118 subTest: Callable 

119 assertNotIn: Callable 

120 

121 def setUp(self) -> None: 

122 self.runner = LogCliRunner() 

123 

124 class PythonLogger: 

125 """Keeps track of log level of a component and number of handlers 

126 attached to it at the time this object was initialized. 

127 """ 

128 

129 def __init__(self, component: str | None) -> None: 

130 self.logger = logging.getLogger(component) 

131 self.initialLevel = self.logger.level 

132 

133 class LsstLogger: 

134 """Keeps track of log level for a component at the time this object was 

135 initialized. 

136 """ 

137 

138 def __init__(self, component: str) -> None: 

139 self.logger = lsstLog.getLogger(component) if lsstLog else None 

140 self.initialLevel = self.logger.getLevel() if lsstLog else None 

141 

142 def runTest(self, cmd: Callable) -> None: 

143 """Test that the log context manager works with the butler cli to 

144 initialize the logging system according to cli inputs for the duration 

145 of the command execution and resets the logging system to its previous 

146 state or expected state when command execution finishes. 

147 """ 

148 pyRoot = self.PythonLogger(None) 

149 pyButler = self.PythonLogger("lsst.daf.butler") 

150 pyLsstRoot = self.PythonLogger("lsst") 

151 lsstRoot = self.LsstLogger("") 

152 lsstButler = self.LsstLogger("lsst.daf.butler") 

153 

154 with command_test_env( 

155 self.runner, "lsst.daf.butler.tests.cliLogTestBase", "command-log-settings-test" 

156 ): 

157 result = cmd() 

158 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

159 

160 # The test environment may have changed the python root logger 

161 # so we can not assume it will be WARNING. 

162 self.assertEqual(pyRoot.logger.level, logging.getLogger().getEffectiveLevel()) 

163 self.assertEqual(pyLsstRoot.logger.level, logging.INFO) 

164 self.assertEqual(pyButler.logger.level, pyButler.initialLevel) 

165 if lsstLog is not None: 

166 self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO) 

167 # lsstLogLevel can either be the initial level, or uninitialized or 

168 # the defined default value. 

169 expectedLsstLogLevel = ( 

170 (lsstButler.initialLevel,) 

171 if lsstButler.initialLevel != -1 

172 else (-1, CliLog.defaultLsstLogLevel) 

173 ) 

174 self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel) 

175 

176 def test_butlerCliLog(self) -> None: 

177 """Test that the log context manager works with the butler cli to 

178 initialize the logging system according to cli inputs for the duration 

179 of the command execution and resets the logging system to its previous 

180 state or expected state when command execution finishes. 

181 """ 

182 # Run with two different log level settings. 

183 log_levels = ( 

184 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

185 # lsstroot, lsstbutler, lsstx 

186 ( 

187 "WARNING", 

188 "lsst.daf.butler=DEBUG", 

189 WARNING, 

190 WARNING, 

191 DEBUG, 

192 lsstLog_WARN, 

193 lsstLog_DEBUG, 

194 WARNING, 

195 ), 

196 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, WARNING), 

197 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, DEBUG), 

198 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

199 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, DEBUG), 

200 ) 

201 

202 self._test_levels(log_levels) 

203 

204 # Check that the environment variable can set additional roots. 

205 log_levels = ( 

206 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

207 # lsstroot, lsstbutler, lsstx 

208 ( 

209 "WARNING", 

210 "lsst.daf.butler=DEBUG", 

211 WARNING, 

212 WARNING, 

213 DEBUG, 

214 lsstLog_WARN, 

215 lsstLog_DEBUG, 

216 WARNING, 

217 ), 

218 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

219 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, INFO), 

220 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

221 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, INFO), 

222 ) 

223 

224 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_ROOT_LOGGER": "lsstx"}): 

225 self._test_levels(log_levels) 

226 

227 def _test_levels(self, log_levels: tuple[tuple[str, str, int, int, int, Any, Any, int], ...]) -> None: 

228 for level1, level2, x_pyroot, x_pylsst, x_pybutler, x_lsstroot, x_lsstbutler, x_lsstx in log_levels: 

229 with self.subTest("Test different log levels", level1=level1, level2=level2): 

230 self.runTest( 

231 partial( 

232 self.runner.invoke, 

233 butlerCli, 

234 [ 

235 "--log-level", 

236 level1, 

237 "--log-level", 

238 level2, 

239 "command-log-settings-test", 

240 "--expected-pyroot-level", 

241 x_pyroot, 

242 "--expected-pylsst-level", 

243 x_pylsst, 

244 "--expected-pybutler-level", 

245 x_pybutler, 

246 "--expected-lsstroot-level", 

247 x_lsstroot, 

248 "--expected-lsstbutler-level", 

249 x_lsstbutler, 

250 "--expected-lsstx-level", 

251 x_lsstx, 

252 ], 

253 ) 

254 ) 

255 

256 def test_helpLogReset(self) -> None: 

257 """Verify that when a command does not execute, like when the help menu 

258 is printed instead, that CliLog is still reset. 

259 """ 

260 self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"])) 

261 

262 def testLongLog(self) -> None: 

263 """Verify the timestamp is in the log messages when the --long-log 

264 flag is set. 

265 """ 

266 # When longlog=True, loglines start with the log level and a 

267 # timestamp with the following format: 

268 # "year-month-day T hour-minute-second.millisecond-zoneoffset" 

269 # For example: "DEBUG 2020-10-28T10:20:31-07:00 ..."" 

270 # The log level name can change, we verify there is an all 

271 # caps word there but do not verify the word. We do not verify 

272 # the rest of the log string, assume that if the timestamp is 

273 # in the string that the rest of the string will appear as 

274 # expected. 

275 timestampRegex = re.compile( 

276 r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})" 

277 "?([-,+][01][0-9]:[034][05]|Z) .*" 

278 ) 

279 

280 # When longlog=False, log lines start with the module name and 

281 # log level, for example: 

282 # lsst.daf.butler.core.config DEBUG: ... 

283 modulesRegex = re.compile(r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*") 

284 

285 with self.runner.isolated_filesystem(): 

286 for longlog in (True, False): 

287 # The pytest log handler interferes with the log configuration 

288 # settings set up by initLog -- therefore test by using 

289 # a subprocess. 

290 args: tuple[str, ...] 

291 if longlog: 

292 args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here") 

293 else: 

294 args = ("butler", "--log-level", "DEBUG", "create", "here") 

295 result = subprocess.run(args, capture_output=True) 

296 # There are cases where the newlines are stripped from the log 

297 # output (like in Jenkins), since we can't depend on newlines 

298 # in log output they are removed here from test output. 

299 output = StringIO(result.stderr.decode().replace("\n", " ")) 

300 startedWithTimestamp = any([timestampRegex.match(line) for line in output.readlines()]) 

301 output.seek(0) 

302 startedWithModule = any(modulesRegex.match(line) for line in output.readlines()) 

303 if longlog: 

304 self.assertTrue( 

305 startedWithTimestamp, msg=f"did not find timestamp in: \n{output.getvalue()}" 

306 ) 

307 self.assertFalse( 

308 startedWithModule, msg=f"found lines starting with module in: \n{output.getvalue()}" 

309 ) 

310 else: 

311 self.assertFalse(startedWithTimestamp, msg=f"found timestamp in: \n{output.getvalue()}") 

312 self.assertTrue( 

313 startedWithModule, 

314 msg=f"did not find lines starting with module in: \n{output.getvalue()}", 

315 ) 

316 

317 def testFileLogging(self) -> None: 

318 """Test --log-file option.""" 

319 with self.runner.isolated_filesystem(): 

320 for i, suffix in enumerate([".json", ".log"]): 

321 # Get a temporary file name and immediately close it 

322 fd = tempfile.NamedTemporaryFile(suffix=suffix) 

323 filename = fd.name 

324 fd.close() 

325 

326 args = ( 

327 "--log-level", 

328 "DEBUG", 

329 "--log-file", 

330 filename, 

331 "--log-label", 

332 "k1=v1,k2=v2", 

333 "--log-label", 

334 "k3=v3", 

335 "create", 

336 f"here{i}", 

337 ) 

338 

339 result = self.runner.invoke(butlerCli, args) 

340 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

341 

342 # Record to test. Test one in the middle that we know is 

343 # a DEBUG message. The first message might come from 

344 # python itself since warnings are redirected to log 

345 # messages. 

346 num = 4 

347 

348 n_records = 5 

349 if suffix == ".json": 

350 records = ButlerLogRecords.from_file(filename) 

351 self.assertGreater(len(records), num) 

352 self.assertEqual(records[num].levelname, "DEBUG", str(records[num])) 

353 self.assertEqual(records[0].MDC, dict(K1="v1", K2="v2", K3="v3")) 

354 

355 self.assertGreater(len(records), n_records) 

356 else: 

357 with open(filename) as filed: 

358 records_text = filed.readlines() 

359 self.assertGreater(len(records_text), num) 

360 self.assertIn("DEBUG", records_text[num], str(records_text[num])) 

361 self.assertNotIn("{", records_text[num], str(records_text[num])) 

362 

363 self.assertGreater(len(records), n_records) 

364 

365 def testLogTty(self) -> None: 

366 """Verify that log output to terminal can be suppressed.""" 

367 with self.runner.isolated_filesystem(): 

368 for log_tty in (True, False): 

369 # The pytest log handler interferes with the log configuration 

370 # settings set up by initLog -- therefore test by using 

371 # a subprocess. 

372 if log_tty: 

373 args = ("butler", "--log-level", "DEBUG", "--log-tty", "create", "here") 

374 else: 

375 args = ("butler", "--log-level", "DEBUG", "--no-log-tty", "create", "here2") 

376 result = subprocess.run(args, capture_output=True) 

377 

378 output = result.stderr.decode() 

379 if log_tty: 

380 self.assertIn("DEBUG", output) 

381 else: 

382 self.assertNotIn("DEBUG", output) 

383 

384 

385if __name__ == "__main__": 

386 unittest.main()