Coverage for tests/test_logging.py: 10%

196 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-27 08:58 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import io 

23import logging 

24import tempfile 

25import unittest 

26from logging import FileHandler, StreamHandler 

27 

28import lsst.utils.logging 

29from lsst.daf.butler.core.logging import ( 

30 ButlerLogRecord, 

31 ButlerLogRecordHandler, 

32 ButlerLogRecords, 

33 ButlerMDC, 

34 JsonLogFormatter, 

35) 

36from lsst.utils.logging import VERBOSE 

37 

38 

39class LoggingTestCase(unittest.TestCase): 

40 """Test we can capture log messages.""" 

41 

42 def setUp(self): 

43 self.handler = ButlerLogRecordHandler() 

44 

45 self.log = lsst.utils.logging.getLogger(self.id()) 

46 self.log.addHandler(self.handler) 

47 

48 def tearDown(self): 

49 if self.handler and self.log: 

50 self.log.removeHandler(self.handler) 

51 ButlerMDC.restore_log_record_factory() 

52 

53 def testRecordCapture(self): 

54 """Test basic log capture and serialization.""" 

55 

56 self.log.setLevel(VERBOSE) 

57 

58 test_messages = ( 

59 (logging.INFO, "This is a log message", True), 

60 (logging.WARNING, "This is a warning message", True), 

61 (logging.DEBUG, "This debug message should not be stored", False), 

62 (VERBOSE, "A verbose message should appear", True), 

63 ) 

64 

65 for level, message, _ in test_messages: 

66 self.log.log(level, message) 

67 

68 expected = [info for info in test_messages if info[2]] 

69 

70 self.assertEqual(len(self.handler.records), len(expected)) 

71 

72 for given, record in zip(expected, self.handler.records): 

73 self.assertEqual(given[0], record.levelno) 

74 self.assertEqual(given[1], record.message) 

75 

76 # Check that we can serialize the records 

77 json = self.handler.records.json() 

78 

79 records = ButlerLogRecords.parse_raw(json) 

80 for original_record, new_record in zip(self.handler.records, records): 

81 self.assertEqual(new_record, original_record) 

82 self.assertEqual(str(records), str(self.handler.records)) 

83 

84 # Create stream form of serialization. 

85 json_stream = "\n".join(record.json() for record in records) 

86 

87 # Also check we can autodetect the format. 

88 for raw in (json, json.encode(), json_stream, json_stream.encode()): 

89 records = ButlerLogRecords.from_raw(json) 

90 self.assertEqual(records, self.handler.records) 

91 

92 for raw in ("", b""): 

93 self.assertEqual(len(ButlerLogRecords.from_raw(raw)), 0) 

94 self.assertEqual(len(ButlerLogRecords.from_stream(io.StringIO())), 0) 

95 

96 # Send bad text to the parser and it should fail (both bytes and str). 

97 bad_text = "x" * 100 

98 

99 # Include short and long values to trigger different code paths 

100 # in error message creation. 

101 for trim in (True, False): 

102 for bad in (bad_text, bad_text.encode()): 

103 bad = bad[:10] if trim else bad 

104 with self.assertRaises(ValueError) as cm: 

105 ButlerLogRecords.from_raw(bad) 

106 if not trim: 

107 self.assertIn("...", str(cm.exception)) 

108 

109 def testRecordsFormatting(self): 

110 

111 self.log.setLevel(logging.DEBUG) 

112 self.log.debug("debug message") 

113 self.log.warning("warning message") 

114 self.log.critical("critical message") 

115 self.log.verbose("verbose message") 

116 

117 self.assertEqual(len(self.handler.records), 4) 

118 

119 format_default = str(self.handler.records) 

120 

121 # Set the format for these records. 

122 self.handler.records.set_log_format("{levelname}") 

123 format_override = str(self.handler.records) 

124 

125 self.assertNotEqual(format_default, format_override) 

126 self.assertEqual(format_override, "DEBUG\nWARNING\nCRITICAL\nVERBOSE") 

127 

128 # Reset the log format and it should match the original text. 

129 self.handler.records.set_log_format(None) 

130 self.assertEqual(str(self.handler.records), format_default) 

131 

132 def testButlerLogRecords(self): 

133 """Test the list-like methods of ButlerLogRecords.""" 

134 

135 self.log.setLevel(logging.INFO) 

136 

137 n_messages = 10 

138 message = "Message #%d" 

139 for counter in range(n_messages): 

140 self.log.info(message, counter) 

141 

142 records = self.handler.records 

143 self.assertEqual(len(records), n_messages) 

144 

145 # Test slicing. 

146 start = 2 

147 end = 6 

148 subset = records[start:end] 

149 self.assertIsInstance(subset, ButlerLogRecords) 

150 self.assertEqual(len(subset), end - start) 

151 self.assertIn(f"#{start}", subset[0].message) 

152 

153 # Reverse the collection. 

154 backwards = list(reversed(records)) 

155 self.assertEqual(len(backwards), len(records)) 

156 self.assertEqual(records[0], backwards[-1]) 

157 

158 # Test some of the collection manipulation methods. 

159 record_0 = records[0] 

160 records.reverse() 

161 self.assertEqual(records[-1], record_0) 

162 self.assertEqual(records.pop(), record_0) 

163 records[0] = record_0 

164 self.assertEqual(records[0], record_0) 

165 len_records = len(records) 

166 records.insert(2, record_0) 

167 self.assertEqual(len(records), len_records + 1) 

168 self.assertEqual(records[0], records[2]) 

169 

170 # Put the subset records back onto the end of the original. 

171 records.extend(subset) 

172 self.assertEqual(len(records), n_messages + len(subset)) 

173 

174 # Test slice for deleting 

175 initial_length = len(records) 

176 start_del = 1 

177 end_del = 3 

178 del records[start_del:end_del] 

179 self.assertEqual(len(records), initial_length - (end_del - start_del)) 

180 

181 records.clear() 

182 self.assertEqual(len(records), 0) 

183 

184 with self.assertRaises(ValueError): 

185 records.append({}) 

186 

187 def testExceptionInfo(self): 

188 

189 self.log.setLevel(logging.DEBUG) 

190 try: 

191 raise RuntimeError("A problem has been encountered.") 

192 except RuntimeError: 

193 self.log.exception("Caught") 

194 

195 self.assertIn("A problem has been encountered", self.handler.records[0].exc_info) 

196 

197 self.log.warning("No exc_info") 

198 self.assertIsNone(self.handler.records[-1].exc_info) 

199 

200 try: 

201 raise RuntimeError("Debug exception log") 

202 except RuntimeError: 

203 self.log.debug("A problem", exc_info=1) 

204 

205 self.assertIn("Debug exception", self.handler.records[-1].exc_info) 

206 

207 def testMDC(self): 

208 """Test that MDC information appears in messages.""" 

209 self.log.setLevel(logging.INFO) 

210 

211 i = 0 

212 self.log.info("Message %d", i) 

213 i += 1 

214 self.assertEqual(self.handler.records[-1].MDC, {}) 

215 

216 ButlerMDC.add_mdc_log_record_factory() 

217 label = "MDC value" 

218 ButlerMDC.MDC("LABEL", label) 

219 self.log.info("Message %d", i) 

220 self.assertEqual(self.handler.records[-1].MDC["LABEL"], label) 

221 

222 # Change the label and check that the previous record does not 

223 # itself change. 

224 ButlerMDC.MDC("LABEL", "dataId") 

225 self.assertEqual(self.handler.records[-1].MDC["LABEL"], label) 

226 

227 # Format a record with MDC. 

228 record = self.handler.records[-1] 

229 

230 # By default the MDC label should not be involved. 

231 self.assertNotIn(label, str(record)) 

232 

233 # But it can be included. 

234 fmt = "x{MDC[LABEL]}" 

235 self.assertEqual(record.format(fmt), "x" + label) 

236 

237 # But can be optional on a record that didn't set it. 

238 self.assertEqual(self.handler.records[0].format(fmt), "x") 

239 

240 # Set an extra MDC entry and include all content. 

241 extra = "extra" 

242 ButlerMDC.MDC("EXTRA", extra) 

243 

244 i += 1 

245 self.log.info("Message %d", i) 

246 formatted = self.handler.records[-1].format("x{MDC} - {message}") 

247 self.assertIn(f"EXTRA={extra}", formatted) 

248 self.assertIn("LABEL=dataId", formatted) 

249 self.assertIn(f"Message {i}", formatted) 

250 

251 # Clear the MDC and ensure that it does not continue to appear 

252 # in messages. 

253 ButlerMDC.MDCRemove("LABEL") 

254 i += 1 

255 self.log.info("Message %d", i) 

256 self.assertEqual(self.handler.records[-1].format(fmt), "x") 

257 self.assertEqual(self.handler.records[-1].format("{message}"), f"Message {i}") 

258 

259 # MDC context manager 

260 fmt = "x{MDC[LABEL]} - {message}" 

261 ButlerMDC.MDC("LABEL", "original") 

262 with ButlerMDC.set_mdc({"LABEL": "test"}): 

263 i += 1 

264 self.log.info("Message %d", i) 

265 self.assertEqual(self.handler.records[-1].format(fmt), f"xtest - Message {i}") 

266 i += 1 

267 self.log.info("Message %d", i) 

268 self.assertEqual(self.handler.records[-1].format(fmt), f"xoriginal - Message {i}") 

269 

270 

271class TestJsonLogging(unittest.TestCase): 

272 def testJsonLogStream(self): 

273 log = logging.getLogger(self.id()) 

274 log.setLevel(logging.INFO) 

275 

276 # Log to a stream and also to a file. 

277 formatter = JsonLogFormatter() 

278 

279 stream = io.StringIO() 

280 stream_handler = StreamHandler(stream) 

281 stream_handler.setFormatter(formatter) 

282 log.addHandler(stream_handler) 

283 

284 file = tempfile.NamedTemporaryFile(suffix=".json") 

285 filename = file.name 

286 file.close() 

287 

288 file_handler = FileHandler(filename) 

289 file_handler.setFormatter(formatter) 

290 log.addHandler(file_handler) 

291 

292 log.info("A message") 

293 log.warning("A warning") 

294 

295 # Add a blank line to the stream to check the parser ignores it. 

296 print(file=stream) 

297 

298 # Rewind the stream and pull messages out of it. 

299 stream.seek(0) 

300 records = ButlerLogRecords.from_stream(stream) 

301 self.assertIsInstance(records[0], ButlerLogRecord) 

302 self.assertEqual(records[0].message, "A message") 

303 self.assertEqual(records[1].levelname, "WARNING") 

304 

305 # Now read from the file. Add two blank lines to test the parser 

306 # will filter them out. 

307 file_handler.close() 

308 

309 with open(filename, "a") as fd: 

310 print(file=fd) 

311 print(file=fd) 

312 

313 file_records = ButlerLogRecords.from_file(filename) 

314 self.assertEqual(file_records, records) 

315 

316 # And read the file again in bytes and text. 

317 for mode in ("rb", "r"): 

318 with open(filename, mode) as fd: 

319 file_records = ButlerLogRecords.from_stream(fd) 

320 self.assertEqual(file_records, records) 

321 fd.seek(0) 

322 file_records = ButlerLogRecords.from_raw(fd.read()) 

323 self.assertEqual(file_records, records) 

324 

325 # Serialize this model to stream. 

326 stream2 = io.StringIO() 

327 print(records.json(), file=stream2) 

328 stream2.seek(0) 

329 stream_records = ButlerLogRecords.from_stream(stream2) 

330 self.assertEqual(stream_records, records) 

331 

332 

333if __name__ == "__main__": 333 ↛ 334line 333 didn't jump to line 334, because the condition on line 333 was never true

334 unittest.main()