Coverage for tests/test_logging.py: 8%
194 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import io
29import logging
30import tempfile
31import unittest
32from logging import FileHandler, StreamHandler
34import lsst.utils.logging
35from lsst.daf.butler.logging import (
36 ButlerLogRecord,
37 ButlerLogRecordHandler,
38 ButlerLogRecords,
39 ButlerMDC,
40 JsonLogFormatter,
41)
42from lsst.utils.logging import VERBOSE
45class LoggingTestCase(unittest.TestCase):
46 """Test we can capture log messages."""
48 def setUp(self):
49 self.handler = ButlerLogRecordHandler()
51 self.log = lsst.utils.logging.getLogger(self.id())
52 self.log.addHandler(self.handler)
54 def tearDown(self):
55 if self.handler and self.log:
56 self.log.removeHandler(self.handler)
57 ButlerMDC.restore_log_record_factory()
59 def testRecordCapture(self):
60 """Test basic log capture and serialization."""
61 self.log.setLevel(VERBOSE)
63 test_messages = (
64 (logging.INFO, "This is a log message", True),
65 (logging.WARNING, "This is a warning message", True),
66 (logging.DEBUG, "This debug message should not be stored", False),
67 (VERBOSE, "A verbose message should appear", True),
68 )
70 for level, message, _ in test_messages:
71 self.log.log(level, message)
73 expected = [info for info in test_messages if info[2]]
75 self.assertEqual(len(self.handler.records), len(expected))
77 for given, record in zip(expected, self.handler.records, strict=True):
78 self.assertEqual(given[0], record.levelno)
79 self.assertEqual(given[1], record.message)
81 # Check that we can serialize the records
82 json = self.handler.records.model_dump_json()
84 records = ButlerLogRecords.model_validate_json(json)
85 for original_record, new_record in zip(self.handler.records, records, strict=True):
86 self.assertEqual(new_record, original_record)
87 self.assertEqual(str(records), str(self.handler.records))
89 # Create stream form of serialization.
90 json_stream = "\n".join(record.model_dump_json() for record in records)
92 # Also check we can autodetect the format.
93 for raw in (json, json.encode(), json_stream, json_stream.encode()):
94 records = ButlerLogRecords.from_raw(raw)
95 self.assertEqual(records, self.handler.records)
97 for raw in ("", b""):
98 self.assertEqual(len(ButlerLogRecords.from_raw(raw)), 0)
99 self.assertEqual(len(ButlerLogRecords.from_stream(io.StringIO())), 0)
101 # Send bad text to the parser and it should fail (both bytes and str).
102 bad_text = "x" * 100
104 # Include short and long values to trigger different code paths
105 # in error message creation.
106 for trim in (True, False):
107 for bad in (bad_text, bad_text.encode()):
108 bad = bad[:10] if trim else bad
109 with self.assertRaises(ValueError) as cm:
110 ButlerLogRecords.from_raw(bad)
111 if not trim:
112 self.assertIn("...", str(cm.exception))
114 def testRecordsFormatting(self):
115 self.log.setLevel(logging.DEBUG)
116 self.log.debug("debug message")
117 self.log.warning("warning message")
118 self.log.critical("critical message")
119 self.log.verbose("verbose message")
121 self.assertEqual(len(self.handler.records), 4)
123 format_default = str(self.handler.records)
125 # Set the format for these records.
126 self.handler.records.set_log_format("{levelname}")
127 format_override = str(self.handler.records)
129 self.assertNotEqual(format_default, format_override)
130 self.assertEqual(format_override, "DEBUG\nWARNING\nCRITICAL\nVERBOSE")
132 # Reset the log format and it should match the original text.
133 self.handler.records.set_log_format(None)
134 self.assertEqual(str(self.handler.records), format_default)
136 def testButlerLogRecords(self):
137 """Test the list-like methods of ButlerLogRecords."""
138 self.log.setLevel(logging.INFO)
140 n_messages = 10
141 message = "Message #%d"
142 for counter in range(n_messages):
143 self.log.info(message, counter)
145 records = self.handler.records
146 self.assertEqual(len(records), n_messages)
148 # Test slicing.
149 start = 2
150 end = 6
151 subset = records[start:end]
152 self.assertIsInstance(subset, ButlerLogRecords)
153 self.assertEqual(len(subset), end - start)
154 self.assertIn(f"#{start}", subset[0].message)
156 # Reverse the collection.
157 backwards = list(reversed(records))
158 self.assertEqual(len(backwards), len(records))
159 self.assertEqual(records[0], backwards[-1])
161 # Test some of the collection manipulation methods.
162 record_0 = records[0]
163 records.reverse()
164 self.assertEqual(records[-1], record_0)
165 self.assertEqual(records.pop(), record_0)
166 records[0] = record_0
167 self.assertEqual(records[0], record_0)
168 len_records = len(records)
169 records.insert(2, record_0)
170 self.assertEqual(len(records), len_records + 1)
171 self.assertEqual(records[0], records[2])
173 # Put the subset records back onto the end of the original.
174 records.extend(subset)
175 self.assertEqual(len(records), n_messages + len(subset))
177 # Test slice for deleting
178 initial_length = len(records)
179 start_del = 1
180 end_del = 3
181 del records[start_del:end_del]
182 self.assertEqual(len(records), initial_length - (end_del - start_del))
184 records.clear()
185 self.assertEqual(len(records), 0)
187 with self.assertRaises(ValueError):
188 records.append({})
190 def testExceptionInfo(self):
191 self.log.setLevel(logging.DEBUG)
192 try:
193 raise RuntimeError("A problem has been encountered.")
194 except RuntimeError:
195 self.log.exception("Caught")
197 self.assertIn("A problem has been encountered", self.handler.records[0].exc_info)
199 self.log.warning("No exc_info")
200 self.assertIsNone(self.handler.records[-1].exc_info)
202 try:
203 raise RuntimeError("Debug exception log")
204 except RuntimeError:
205 self.log.debug("A problem", exc_info=1)
207 self.assertIn("Debug exception", self.handler.records[-1].exc_info)
209 def testMDC(self):
210 """Test that MDC information appears in messages."""
211 self.log.setLevel(logging.INFO)
213 i = 0
214 self.log.info("Message %d", i)
215 i += 1
216 self.assertEqual(self.handler.records[-1].MDC, {})
218 ButlerMDC.add_mdc_log_record_factory()
219 label = "MDC value"
220 ButlerMDC.MDC("LABEL", label)
221 self.log.info("Message %d", i)
222 self.assertEqual(self.handler.records[-1].MDC["LABEL"], label)
224 # Change the label and check that the previous record does not
225 # itself change.
226 ButlerMDC.MDC("LABEL", "dataId")
227 self.assertEqual(self.handler.records[-1].MDC["LABEL"], label)
229 # Format a record with MDC.
230 record = self.handler.records[-1]
232 # By default the MDC label should not be involved.
233 self.assertNotIn(label, str(record))
235 # But it can be included.
236 fmt = "x{MDC[LABEL]}"
237 self.assertEqual(record.format(fmt), "x" + label)
239 # But can be optional on a record that didn't set it.
240 self.assertEqual(self.handler.records[0].format(fmt), "x")
242 # Set an extra MDC entry and include all content.
243 extra = "extra"
244 ButlerMDC.MDC("EXTRA", extra)
246 i += 1
247 self.log.info("Message %d", i)
248 formatted = self.handler.records[-1].format("x{MDC} - {message}")
249 self.assertIn(f"EXTRA={extra}", formatted)
250 self.assertIn("LABEL=dataId", formatted)
251 self.assertIn(f"Message {i}", formatted)
253 # Clear the MDC and ensure that it does not continue to appear
254 # in messages.
255 ButlerMDC.MDCRemove("LABEL")
256 i += 1
257 self.log.info("Message %d", i)
258 self.assertEqual(self.handler.records[-1].format(fmt), "x")
259 self.assertEqual(self.handler.records[-1].format("{message}"), f"Message {i}")
261 # MDC context manager
262 fmt = "x{MDC[LABEL]} - {message}"
263 ButlerMDC.MDC("LABEL", "original")
264 with ButlerMDC.set_mdc({"LABEL": "test"}):
265 i += 1
266 self.log.info("Message %d", i)
267 self.assertEqual(self.handler.records[-1].format(fmt), f"xtest - Message {i}")
268 i += 1
269 self.log.info("Message %d", i)
270 self.assertEqual(self.handler.records[-1].format(fmt), f"xoriginal - Message {i}")
273class TestJsonLogging(unittest.TestCase):
274 """Test logging using JSON."""
276 def testJsonLogStream(self):
277 log = logging.getLogger(self.id())
278 log.setLevel(logging.INFO)
280 # Log to a stream and also to a file.
281 formatter = JsonLogFormatter()
283 stream = io.StringIO()
284 stream_handler = StreamHandler(stream)
285 stream_handler.setFormatter(formatter)
286 log.addHandler(stream_handler)
288 file = tempfile.NamedTemporaryFile(suffix=".json")
289 filename = file.name
290 file.close()
292 file_handler = FileHandler(filename)
293 file_handler.setFormatter(formatter)
294 log.addHandler(file_handler)
296 log.info("A message")
297 log.warning("A warning")
299 # Add a blank line to the stream to check the parser ignores it.
300 print(file=stream)
302 # Rewind the stream and pull messages out of it.
303 stream.seek(0)
304 records = ButlerLogRecords.from_stream(stream)
305 self.assertIsInstance(records[0], ButlerLogRecord)
306 self.assertEqual(records[0].message, "A message")
307 self.assertEqual(records[1].levelname, "WARNING")
309 # Now read from the file. Add two blank lines to test the parser
310 # will filter them out.
311 file_handler.close()
313 with open(filename, "a") as fd:
314 print(file=fd)
315 print(file=fd)
317 file_records = ButlerLogRecords.from_file(filename)
318 self.assertEqual(file_records, records)
320 # And read the file again in bytes and text.
321 for mode in ("rb", "r"):
322 with open(filename, mode) as fd:
323 file_records = ButlerLogRecords.from_stream(fd)
324 self.assertEqual(file_records, records)
325 fd.seek(0)
326 file_records = ButlerLogRecords.from_raw(fd.read())
327 self.assertEqual(file_records, records)
329 # Serialize this model to stream.
330 stream2 = io.StringIO()
331 print(records.model_dump_json(), file=stream2)
332 stream2.seek(0)
333 stream_records = ButlerLogRecords.from_stream(stream2)
334 self.assertEqual(stream_records, records)
337if __name__ == "__main__":
338 unittest.main()