Coverage for python/lsst/daf/butler/tests/cliLogTestBase.py: 26%

146 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-14 02:05 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23"""Unit tests for the daf_butler CliLog utility. 

24 

25Code is implemented in 

26daf_butler but some only runs if lsst.log.Log can be imported so these parts of 

27it can't be tested there because daf_butler does not directly depend on 

28lsst.log, and only uses it if it has been setup by another package.""" 

29 

30import logging 

31import os 

32import re 

33import subprocess 

34import tempfile 

35import unittest 

36from collections import namedtuple 

37from collections.abc import Callable 

38from functools import partial 

39from io import StringIO 

40from logging import DEBUG, INFO, WARNING 

41from typing import TYPE_CHECKING, Any 

42 

43import click 

44from lsst.daf.butler.cli.butler import cli as butlerCli 

45from lsst.daf.butler.cli.cliLog import CliLog 

46from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, command_test_env 

47from lsst.daf.butler.core.logging import ButlerLogRecords 

48from lsst.utils.logging import TRACE 

49 

50try: 

51 import lsst.log as lsstLog 

52 

53 lsstLog_INFO = lsstLog.INFO 

54 lsstLog_DEBUG = lsstLog.DEBUG 

55 lsstLog_WARN = lsstLog.WARN 

56except ModuleNotFoundError: 

57 lsstLog = None 

58 lsstLog_INFO = 0 

59 lsstLog_DEBUG = 0 

60 lsstLog_WARN = 0 

61 

62 

63@click.command() 

64@click.option("--expected-pyroot-level", type=int) 

65@click.option("--expected-pylsst-level", type=int) 

66@click.option("--expected-pybutler-level", type=int) 

67@click.option("--expected-lsstroot-level", type=int) 

68@click.option("--expected-lsstbutler-level", type=int) 

69@click.option("--expected-lsstx-level", type=int) 

70def command_log_settings_test( 

71 expected_pyroot_level: str, 

72 expected_pylsst_level: str, 

73 expected_pybutler_level: str, 

74 expected_lsstroot_level: str, 

75 expected_lsstbutler_level: str, 

76 expected_lsstx_level: str, 

77) -> None: 

78 LogLevel = namedtuple("LogLevel", ("expected", "actual", "name")) 

79 

80 logLevels = [ 

81 LogLevel(expected_pyroot_level, logging.getLogger().level, "pyRoot"), 

82 LogLevel(expected_pylsst_level, logging.getLogger("lsst").getEffectiveLevel(), "pyLsst"), 

83 LogLevel( 

84 expected_pybutler_level, logging.getLogger("lsst.daf.butler").getEffectiveLevel(), "pyButler" 

85 ), 

86 LogLevel(expected_lsstx_level, logging.getLogger("lsstx").getEffectiveLevel(), "pyLsstx"), 

87 ] 

88 if lsstLog is not None: 

89 logLevels.extend( 

90 [ 

91 LogLevel(expected_lsstroot_level, lsstLog.getLogger("lsst").getEffectiveLevel(), "lsstRoot"), 

92 LogLevel( 

93 expected_lsstbutler_level, 

94 lsstLog.getLogger("lsst.daf.butler").getEffectiveLevel(), 

95 "lsstButler", 

96 ), 

97 ] 

98 ) 

99 for expected, actual, name in logLevels: 

100 if expected != actual: 

101 raise ( 

102 click.ClickException(message=f"expected {name} level to be {expected!r}, actual:{actual!r}") 

103 ) 

104 

105 

106class CliLogTestBase: 

107 """Tests log initialization, reset, and setting log levels.""" 

108 

109 if TYPE_CHECKING: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true

110 assertEqual: Callable 

111 assertIn: Callable 

112 assertTrue: Callable 

113 assertFalse: Callable 

114 assertGreater: Callable 

115 subTest: Callable 

116 assertNotIn: Callable 

117 

118 def setUp(self) -> None: 

119 self.runner = LogCliRunner() 

120 

121 class PythonLogger: 

122 """Keeps track of log level of a component and number of handlers 

123 attached to it at the time this object was initialized.""" 

124 

125 def __init__(self, component: str | None) -> None: 

126 self.logger = logging.getLogger(component) 

127 self.initialLevel = self.logger.level 

128 

129 class LsstLogger: 

130 """Keeps track of log level for a component at the time this object was 

131 initialized.""" 

132 

133 def __init__(self, component: str) -> None: 

134 self.logger = lsstLog.getLogger(component) if lsstLog else None 

135 self.initialLevel = self.logger.getLevel() if lsstLog else None 

136 

137 def runTest(self, cmd: Callable) -> None: 

138 """Test that the log context manager works with the butler cli to 

139 initialize the logging system according to cli inputs for the duration 

140 of the command execution and resets the logging system to its previous 

141 state or expected state when command execution finishes.""" 

142 pyRoot = self.PythonLogger(None) 

143 pyButler = self.PythonLogger("lsst.daf.butler") 

144 pyLsstRoot = self.PythonLogger("lsst") 

145 lsstRoot = self.LsstLogger("") 

146 lsstButler = self.LsstLogger("lsst.daf.butler") 

147 

148 with command_test_env( 

149 self.runner, "lsst.daf.butler.tests.cliLogTestBase", "command-log-settings-test" 

150 ): 

151 result = cmd() 

152 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

153 

154 self.assertEqual(pyRoot.logger.level, logging.WARNING) 

155 self.assertEqual(pyLsstRoot.logger.level, logging.INFO) 

156 self.assertEqual(pyButler.logger.level, pyButler.initialLevel) 

157 if lsstLog is not None: 

158 self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO) 

159 # lsstLogLevel can either be the initial level, or uninitialized or 

160 # the defined default value. 

161 expectedLsstLogLevel = ( 

162 (lsstButler.initialLevel,) 

163 if lsstButler.initialLevel != -1 

164 else (-1, CliLog.defaultLsstLogLevel) 

165 ) 

166 self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel) 

167 

168 def test_butlerCliLog(self) -> None: 

169 """Test that the log context manager works with the butler cli to 

170 initialize the logging system according to cli inputs for the duration 

171 of the command execution and resets the logging system to its previous 

172 state or expected state when command execution finishes.""" 

173 

174 # Run with two different log level settings. 

175 log_levels = ( 

176 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

177 # lsstroot, lsstbutler, lsstx 

178 ( 

179 "WARNING", 

180 "lsst.daf.butler=DEBUG", 

181 WARNING, 

182 WARNING, 

183 DEBUG, 

184 lsstLog_WARN, 

185 lsstLog_DEBUG, 

186 WARNING, 

187 ), 

188 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, WARNING), 

189 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, DEBUG), 

190 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

191 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, DEBUG), 

192 ) 

193 

194 self._test_levels(log_levels) 

195 

196 # Check that the environment variable can set additional roots. 

197 log_levels = ( 

198 # --log-level / --log-level / expected pyroot, pylsst, pybutler, 

199 # lsstroot, lsstbutler, lsstx 

200 ( 

201 "WARNING", 

202 "lsst.daf.butler=DEBUG", 

203 WARNING, 

204 WARNING, 

205 DEBUG, 

206 lsstLog_WARN, 

207 lsstLog_DEBUG, 

208 WARNING, 

209 ), 

210 ("DEBUG", "lsst.daf.butler=TRACE", WARNING, DEBUG, TRACE, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

211 (".=DEBUG", "lsst.daf.butler=WARNING", DEBUG, INFO, WARNING, lsstLog_INFO, lsstLog_WARN, INFO), 

212 (".=DEBUG", "DEBUG", DEBUG, DEBUG, DEBUG, lsstLog_DEBUG, lsstLog_DEBUG, DEBUG), 

213 (".=DEBUG", "conda=DEBUG", DEBUG, INFO, INFO, lsstLog_INFO, lsstLog_INFO, INFO), 

214 ) 

215 

216 with unittest.mock.patch.dict(os.environ, {"DAF_BUTLER_ROOT_LOGGER": "lsstx"}): 

217 self._test_levels(log_levels) 

218 

219 def _test_levels(self, log_levels: tuple[tuple[str, str, int, int, int, Any, Any, int], ...]) -> None: 

220 for level1, level2, x_pyroot, x_pylsst, x_pybutler, x_lsstroot, x_lsstbutler, x_lsstx in log_levels: 

221 with self.subTest("Test different log levels", level1=level1, level2=level2): 

222 self.runTest( 

223 partial( 

224 self.runner.invoke, 

225 butlerCli, 

226 [ 

227 "--log-level", 

228 level1, 

229 "--log-level", 

230 level2, 

231 "command-log-settings-test", 

232 "--expected-pyroot-level", 

233 x_pyroot, 

234 "--expected-pylsst-level", 

235 x_pylsst, 

236 "--expected-pybutler-level", 

237 x_pybutler, 

238 "--expected-lsstroot-level", 

239 x_lsstroot, 

240 "--expected-lsstbutler-level", 

241 x_lsstbutler, 

242 "--expected-lsstx-level", 

243 x_lsstx, 

244 ], 

245 ) 

246 ) 

247 

248 def test_helpLogReset(self) -> None: 

249 """Verify that when a command does not execute, like when the help menu 

250 is printed instead, that CliLog is still reset.""" 

251 

252 self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"])) 

253 

254 def testLongLog(self) -> None: 

255 """Verify the timestamp is in the log messages when the --long-log 

256 flag is set.""" 

257 

258 # When longlog=True, loglines start with the log level and a 

259 # timestamp with the following format: 

260 # "year-month-day T hour-minute-second.millisecond-zoneoffset" 

261 # For example: "DEBUG 2020-10-28T10:20:31-07:00 ..."" 

262 # The log level name can change, we verify there is an all 

263 # caps word there but do not verify the word. We do not verify 

264 # the rest of the log string, assume that if the timestamp is 

265 # in the string that the rest of the string will appear as 

266 # expected. 

267 timestampRegex = re.compile( 

268 r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})" 

269 "?([-,+][01][0-9]:[034][05]|Z) .*" 

270 ) 

271 

272 # When longlog=False, log lines start with the module name and 

273 # log level, for example: 

274 # lsst.daf.butler.core.config DEBUG: ... 

275 modulesRegex = re.compile(r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*") 

276 

277 with self.runner.isolated_filesystem(): 

278 for longlog in (True, False): 

279 # The pytest log handler interferes with the log configuration 

280 # settings set up by initLog -- therefore test by using 

281 # a subprocess. 

282 args: tuple[str, ...] 

283 if longlog: 

284 args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here") 

285 else: 

286 args = ("butler", "--log-level", "DEBUG", "create", "here") 

287 result = subprocess.run(args, capture_output=True) 

288 # There are cases where the newlines are stripped from the log 

289 # output (like in Jenkins), since we can't depend on newlines 

290 # in log output they are removed here from test output. 

291 output = StringIO((result.stderr.decode().replace("\n", " "))) 

292 startedWithTimestamp = any([timestampRegex.match(line) for line in output.readlines()]) 

293 output.seek(0) 

294 startedWithModule = any(modulesRegex.match(line) for line in output.readlines()) 

295 if longlog: 

296 self.assertTrue( 

297 startedWithTimestamp, msg=f"did not find timestamp in: \n{output.getvalue()}" 

298 ) 

299 self.assertFalse( 

300 startedWithModule, msg=f"found lines starting with module in: \n{output.getvalue()}" 

301 ) 

302 else: 

303 self.assertFalse(startedWithTimestamp, msg=f"found timestamp in: \n{output.getvalue()}") 

304 self.assertTrue( 

305 startedWithModule, 

306 msg=f"did not find lines starting with module in: \n{output.getvalue()}", 

307 ) 

308 

309 def testFileLogging(self) -> None: 

310 """Test --log-file option.""" 

311 with self.runner.isolated_filesystem(): 

312 for i, suffix in enumerate([".json", ".log"]): 

313 # Get a temporary file name and immediately close it 

314 fd = tempfile.NamedTemporaryFile(suffix=suffix) 

315 filename = fd.name 

316 fd.close() 

317 

318 args = ( 

319 "--log-level", 

320 "DEBUG", 

321 "--log-file", 

322 filename, 

323 "--log-label", 

324 "k1=v1,k2=v2", 

325 "--log-label", 

326 "k3=v3", 

327 "create", 

328 f"here{i}", 

329 ) 

330 

331 result = self.runner.invoke(butlerCli, args) 

332 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

333 

334 # Record to test. Test one in the middle that we know is 

335 # a DEBUG message. The first message might come from 

336 # python itself since warnings are redirected to log 

337 # messages. 

338 num = 4 

339 

340 n_records = 5 

341 if suffix == ".json": 

342 records = ButlerLogRecords.from_file(filename) 

343 self.assertGreater(len(records), num) 

344 self.assertEqual(records[num].levelname, "DEBUG", str(records[num])) 

345 self.assertEqual(records[0].MDC, dict(K1="v1", K2="v2", K3="v3")) 

346 

347 self.assertGreater(len(records), n_records) 

348 else: 

349 with open(filename) as filed: 

350 records_text = filed.readlines() 

351 self.assertGreater(len(records_text), num) 

352 self.assertIn("DEBUG", records_text[num], str(records_text[num])) 

353 self.assertNotIn("{", records_text[num], str(records_text[num])) 

354 

355 self.assertGreater(len(records), n_records) 

356 

357 def testLogTty(self) -> None: 

358 """Verify that log output to terminal can be suppressed.""" 

359 

360 with self.runner.isolated_filesystem(): 

361 for log_tty in (True, False): 

362 # The pytest log handler interferes with the log configuration 

363 # settings set up by initLog -- therefore test by using 

364 # a subprocess. 

365 if log_tty: 

366 args = ("butler", "--log-level", "DEBUG", "--log-tty", "create", "here") 

367 else: 

368 args = ("butler", "--log-level", "DEBUG", "--no-log-tty", "create", "here2") 

369 result = subprocess.run(args, capture_output=True) 

370 

371 output = result.stderr.decode() 

372 if log_tty: 

373 self.assertIn("DEBUG", output) 

374 else: 

375 self.assertNotIn("DEBUG", output) 

376 

377 

378if __name__ == "__main__": 378 ↛ 379line 378 didn't jump to line 379, because the condition on line 378 was never true

379 unittest.main()