Coverage for tests/test_logging.py: 8%
195 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-13 10:57 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-13 10:57 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import io
29import logging
30import tempfile
31import unittest
32from logging import FileHandler, StreamHandler
34import lsst.utils.logging
35from lsst.daf.butler.logging import (
36 ButlerLogRecord,
37 ButlerLogRecordHandler,
38 ButlerLogRecords,
39 ButlerMDC,
40 JsonLogFormatter,
41)
42from lsst.utils.logging import VERBOSE
45class LoggingTestCase(unittest.TestCase):
46 """Test we can capture log messages."""
48 def setUp(self):
49 self.handler = ButlerLogRecordHandler()
51 self.log = lsst.utils.logging.getLogger(self.id())
52 self.log.addHandler(self.handler)
54 def tearDown(self):
55 if self.handler and self.log:
56 self.log.removeHandler(self.handler)
57 ButlerMDC.restore_log_record_factory()
58 ButlerMDC.clear_mdc()
60 def testRecordCapture(self):
61 """Test basic log capture and serialization."""
62 self.log.setLevel(VERBOSE)
64 test_messages = (
65 (logging.INFO, "This is a log message", True),
66 (logging.WARNING, "This is a warning message", True),
67 (logging.DEBUG, "This debug message should not be stored", False),
68 (VERBOSE, "A verbose message should appear", True),
69 )
71 for level, message, _ in test_messages:
72 self.log.log(level, message)
74 expected = [info for info in test_messages if info[2]]
76 self.assertEqual(len(self.handler.records), len(expected))
78 for given, record in zip(expected, self.handler.records, strict=True):
79 self.assertEqual(given[0], record.levelno)
80 self.assertEqual(given[1], record.message)
82 # Check that we can serialize the records
83 json = self.handler.records.model_dump_json()
85 records = ButlerLogRecords.model_validate_json(json)
86 for original_record, new_record in zip(self.handler.records, records, strict=True):
87 self.assertEqual(new_record, original_record)
88 self.assertEqual(str(records), str(self.handler.records))
90 # Create stream form of serialization.
91 json_stream = "\n".join(record.model_dump_json() for record in records)
93 # Also check we can autodetect the format.
94 for raw in (json, json.encode(), json_stream, json_stream.encode()):
95 records = ButlerLogRecords.from_raw(raw)
96 self.assertEqual(records, self.handler.records)
98 for raw in ("", b""):
99 self.assertEqual(len(ButlerLogRecords.from_raw(raw)), 0)
100 self.assertEqual(len(ButlerLogRecords.from_stream(io.StringIO())), 0)
102 # Send bad text to the parser and it should fail (both bytes and str).
103 bad_text = "x" * 100
105 # Include short and long values to trigger different code paths
106 # in error message creation.
107 for trim in (True, False):
108 for bad in (bad_text, bad_text.encode()):
109 bad = bad[:10] if trim else bad
110 with self.assertRaises(ValueError) as cm:
111 ButlerLogRecords.from_raw(bad)
112 if not trim:
113 self.assertIn("...", str(cm.exception))
115 def testRecordsFormatting(self):
116 self.log.setLevel(logging.DEBUG)
117 self.log.debug("debug message")
118 self.log.warning("warning message")
119 self.log.critical("critical message")
120 self.log.verbose("verbose message")
122 self.assertEqual(len(self.handler.records), 4)
124 format_default = str(self.handler.records)
126 # Set the format for these records.
127 self.handler.records.set_log_format("{levelname}")
128 format_override = str(self.handler.records)
130 self.assertNotEqual(format_default, format_override)
131 self.assertEqual(format_override, "DEBUG\nWARNING\nCRITICAL\nVERBOSE")
133 # Reset the log format and it should match the original text.
134 self.handler.records.set_log_format(None)
135 self.assertEqual(str(self.handler.records), format_default)
137 def testButlerLogRecords(self):
138 """Test the list-like methods of ButlerLogRecords."""
139 self.log.setLevel(logging.INFO)
141 n_messages = 10
142 message = "Message #%d"
143 for counter in range(n_messages):
144 self.log.info(message, counter)
146 records = self.handler.records
147 self.assertEqual(len(records), n_messages)
149 # Test slicing.
150 start = 2
151 end = 6
152 subset = records[start:end]
153 self.assertIsInstance(subset, ButlerLogRecords)
154 self.assertEqual(len(subset), end - start)
155 self.assertIn(f"#{start}", subset[0].message)
157 # Reverse the collection.
158 backwards = list(reversed(records))
159 self.assertEqual(len(backwards), len(records))
160 self.assertEqual(records[0], backwards[-1])
162 # Test some of the collection manipulation methods.
163 record_0 = records[0]
164 records.reverse()
165 self.assertEqual(records[-1], record_0)
166 self.assertEqual(records.pop(), record_0)
167 records[0] = record_0
168 self.assertEqual(records[0], record_0)
169 len_records = len(records)
170 records.insert(2, record_0)
171 self.assertEqual(len(records), len_records + 1)
172 self.assertEqual(records[0], records[2])
174 # Put the subset records back onto the end of the original.
175 records.extend(subset)
176 self.assertEqual(len(records), n_messages + len(subset))
178 # Test slice for deleting
179 initial_length = len(records)
180 start_del = 1
181 end_del = 3
182 del records[start_del:end_del]
183 self.assertEqual(len(records), initial_length - (end_del - start_del))
185 records.clear()
186 self.assertEqual(len(records), 0)
188 with self.assertRaises(ValueError):
189 records.append({})
191 def testExceptionInfo(self):
192 self.log.setLevel(logging.DEBUG)
193 try:
194 raise RuntimeError("A problem has been encountered.")
195 except RuntimeError:
196 self.log.exception("Caught")
198 self.assertIn("A problem has been encountered", self.handler.records[0].exc_info)
200 self.log.warning("No exc_info")
201 self.assertIsNone(self.handler.records[-1].exc_info)
203 try:
204 raise RuntimeError("Debug exception log")
205 except RuntimeError:
206 self.log.debug("A problem", exc_info=1)
208 self.assertIn("Debug exception", self.handler.records[-1].exc_info)
210 def testMDC(self):
211 """Test that MDC information appears in messages."""
212 self.log.setLevel(logging.INFO)
214 i = 0
215 self.log.info("Message %d", i)
216 i += 1
217 self.assertEqual(self.handler.records[-1].MDC, {})
219 ButlerMDC.add_mdc_log_record_factory()
220 label = "MDC value"
221 ButlerMDC.MDC("LABEL", label)
222 self.log.info("Message %d", i)
223 self.assertEqual(self.handler.records[-1].MDC["LABEL"], label)
225 # Change the label and check that the previous record does not
226 # itself change.
227 ButlerMDC.MDC("LABEL", "dataId")
228 self.assertEqual(self.handler.records[-1].MDC["LABEL"], label)
230 # Format a record with MDC.
231 record = self.handler.records[-1]
233 # By default the MDC label should not be involved.
234 self.assertNotIn(label, str(record))
236 # But it can be included.
237 fmt = "x{MDC[LABEL]}"
238 self.assertEqual(record.format(fmt), "x" + label)
240 # But can be optional on a record that didn't set it.
241 self.assertEqual(self.handler.records[0].format(fmt), "x")
243 # Set an extra MDC entry and include all content.
244 extra = "extra"
245 ButlerMDC.MDC("EXTRA", extra)
247 i += 1
248 self.log.info("Message %d", i)
249 formatted = self.handler.records[-1].format("x{MDC} - {message}")
250 self.assertIn(f"EXTRA={extra}", formatted)
251 self.assertIn("LABEL=dataId", formatted)
252 self.assertIn(f"Message {i}", formatted)
254 # Clear the MDC and ensure that it does not continue to appear
255 # in messages.
256 ButlerMDC.MDCRemove("LABEL")
257 i += 1
258 self.log.info("Message %d", i)
259 self.assertEqual(self.handler.records[-1].format(fmt), "x")
260 self.assertEqual(self.handler.records[-1].format("{message}"), f"Message {i}")
262 # MDC context manager
263 fmt = "x{MDC[LABEL]} - {message}"
264 ButlerMDC.MDC("LABEL", "original")
265 with ButlerMDC.set_mdc({"LABEL": "test"}):
266 i += 1
267 self.log.info("Message %d", i)
268 self.assertEqual(self.handler.records[-1].format(fmt), f"xtest - Message {i}")
269 i += 1
270 self.log.info("Message %d", i)
271 self.assertEqual(self.handler.records[-1].format(fmt), f"xoriginal - Message {i}")
274class TestJsonLogging(unittest.TestCase):
275 """Test logging using JSON."""
277 def testJsonLogStream(self):
278 log = logging.getLogger(self.id())
279 log.setLevel(logging.INFO)
281 # Log to a stream and also to a file.
282 formatter = JsonLogFormatter()
284 stream = io.StringIO()
285 stream_handler = StreamHandler(stream)
286 stream_handler.setFormatter(formatter)
287 log.addHandler(stream_handler)
289 file = tempfile.NamedTemporaryFile(suffix=".json")
290 filename = file.name
291 file.close()
293 file_handler = FileHandler(filename)
294 file_handler.setFormatter(formatter)
295 log.addHandler(file_handler)
297 log.info("A message")
298 log.warning("A warning")
300 # Add a blank line to the stream to check the parser ignores it.
301 print(file=stream)
303 # Rewind the stream and pull messages out of it.
304 stream.seek(0)
305 records = ButlerLogRecords.from_stream(stream)
306 self.assertIsInstance(records[0], ButlerLogRecord)
307 self.assertEqual(records[0].message, "A message")
308 self.assertEqual(records[1].levelname, "WARNING")
310 # Now read from the file. Add two blank lines to test the parser
311 # will filter them out.
312 file_handler.close()
314 with open(filename, "a") as fd:
315 print(file=fd)
316 print(file=fd)
318 file_records = ButlerLogRecords.from_file(filename)
319 self.assertEqual(file_records, records)
321 # And read the file again in bytes and text.
322 for mode in ("rb", "r"):
323 with open(filename, mode) as fd:
324 file_records = ButlerLogRecords.from_stream(fd)
325 self.assertEqual(file_records, records)
326 fd.seek(0)
327 file_records = ButlerLogRecords.from_raw(fd.read())
328 self.assertEqual(file_records, records)
330 # Serialize this model to stream.
331 stream2 = io.StringIO()
332 print(records.model_dump_json(), file=stream2)
333 stream2.seek(0)
334 stream_records = ButlerLogRecords.from_stream(stream2)
335 self.assertEqual(stream_records, records)
338if __name__ == "__main__":
339 unittest.main()