Coverage for tests/test_log.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2# LSST Data Management System
3# Copyright 2014-2017 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
22"""
23This tests the logging system in a variety of ways.
24"""
27import os
28import shutil
29import tempfile
30import threading
31import unittest
32import logging
33import lsst.log as log
36class TestLog(unittest.TestCase):
38 class StdoutCapture(object):
39 """
40 Context manager to redirect stdout to a file.
41 """
43 def __init__(self, filename):
44 self.stdout = None
45 self.outputFilename = filename
47 def __enter__(self):
48 self.stdout = os.dup(1)
49 os.close(1)
50 os.open(self.outputFilename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
52 def __exit__(self, type, value, traceback):
53 if self.stdout is not None:
54 os.close(1)
55 os.dup(self.stdout)
56 os.close(self.stdout)
57 self.stdout = None
59 def setUp(self):
60 """Make a temporary directory and a log file in it."""
61 self.tempDir = tempfile.mkdtemp()
62 self.outputFilename = os.path.join(self.tempDir, "log.out")
63 self.stdout = None
65 def tearDown(self):
66 """Remove the temporary directory and clean up Python forwarding."""
67 log.doNotUsePythonLogging()
68 shutil.rmtree(self.tempDir)
70 def configure(self, configuration):
71 """
72 Create a configuration file in the temporary directory and populate
73 it with the provided string.
74 """
75 log.configure_prop(configuration.format(self.outputFilename))
77 def check(self, reference):
78 """Compare the log file with the provided reference text."""
79 with open(self.outputFilename, 'r') as f:
80 # strip everything up to first ] to remove timestamp and thread ID
81 lines = [line.split(']')[-1].rstrip("\n") for line in f.readlines()]
82 reflines = [line for line in reference.split("\n") if line != ""]
83 self.maxDiff = None
84 self.assertListEqual(lines, reflines)
86 def testDefaultLogger(self):
87 """Check the default root logger name."""
88 self.assertEqual(log.getDefaultLogger().getName(), "")
90 def testBasic(self):
91 """
92 Test basic log output with default configuration.
93 Since the default threshold is INFO, the DEBUG or TRACE
94 message is not emitted.
95 """
96 with TestLog.StdoutCapture(self.outputFilename):
97 log.configure()
98 log.log(log.getDefaultLogger(), log.INFO, "This is INFO")
99 log.info(u"This is unicode INFO")
100 log.trace("This is TRACE")
101 log.debug("This is DEBUG")
102 log.warn("This is WARN")
103 log.error("This is ERROR")
104 log.fatal("This is FATAL")
105 log.warning("Format %d %g %s", 3, 2.71828, "foo")
106 self.check("""
107root INFO: This is INFO
108root INFO: This is unicode INFO
109root WARN: This is WARN
110root ERROR: This is ERROR
111root FATAL: This is FATAL
112root WARN: Format 3 2.71828 foo
113""")
115 def testBasicFormat(self):
116 """
117 Test basic log output with default configuration but using
118 the f variants.
119 Since the default threshold is INFO, the DEBUG or TRACE
120 message is not emitted.
121 """
122 with TestLog.StdoutCapture(self.outputFilename):
123 log.configure()
124 log.logf(log.getDefaultLogger(), log.INFO,
125 "This is {{INFO}} Item 1: {item[1]}",
126 item=["a", "b", "c"])
127 log.infof(u"This is {unicode} INFO")
128 log.tracef("This is TRACE")
129 log.debugf("This is DEBUG")
130 log.warnf("This is WARN {city}", city="Tucson")
131 log.errorf("This is ERROR {1}->{0}", 2, 1)
132 log.fatalf("This is FATAL {1} out of {0} times for {place}",
133 4, 3, place="LSST")
134 log.warnf("Format {} {} {}", 3, 2.71828, "foo")
135 self.check("""
136root INFO: This is {INFO} Item 1: b
137root INFO: This is {unicode} INFO
138root WARN: This is WARN Tucson
139root ERROR: This is ERROR 1->2
140root FATAL: This is FATAL 3 out of 4 times for LSST
141root WARN: Format 3 2.71828 foo
142""")
144 def testPattern(self):
145 """
146 Test a complex pattern for log messages, including Mapped
147 Diagnostic Context (MDC).
148 """
149 with TestLog.StdoutCapture(self.outputFilename):
150 self.configure("""
151log4j.rootLogger=DEBUG, CA
152log4j.appender.CA=ConsoleAppender
153log4j.appender.CA.layout=PatternLayout
154log4j.appender.CA.layout.ConversionPattern=%-5p %c %C %M (%F:%L) %l - %m - %X%n
155""")
156 log.trace("This is TRACE")
157 log.info("This is INFO")
158 log.debug("This is DEBUG")
160 log.MDC("x", 3)
161 log.MDC("y", "foo")
162 log.MDC("z", TestLog)
164 log.trace("This is TRACE 2")
165 log.info("This is INFO 2")
166 log.debug("This is DEBUG 2")
167 log.MDCRemove("z")
169 log.trace("This is TRACE 3")
170 log.info("This is INFO 3")
171 log.debug("This is DEBUG 3")
172 log.MDCRemove("x")
173 log.trace("This is TRACE 4")
174 log.info("This is INFO 4")
175 log.debug("This is DEBUG 4")
177 log.trace("This is TRACE 5")
178 log.info("This is INFO 5")
179 log.debug("This is DEBUG 5")
181 log.MDCRemove("y")
183 # Use format to make line numbers easier to change.
184 self.check("""
185INFO root testPattern (test_log.py:{0[0]}) test_log.py({0[0]}) - This is INFO - {{}}
186DEBUG root testPattern (test_log.py:{0[1]}) test_log.py({0[1]}) - This is DEBUG - {{}}
187INFO root testPattern (test_log.py:{0[2]}) test_log.py({0[2]}) - This is INFO 2 - {{{{x,3}}{{y,foo}}{{z,<class '{1}.TestLog'>}}}}
188DEBUG root testPattern (test_log.py:{0[3]}) test_log.py({0[3]}) - This is DEBUG 2 - {{{{x,3}}{{y,foo}}{{z,<class '{1}.TestLog'>}}}}
189INFO root testPattern (test_log.py:{0[4]}) test_log.py({0[4]}) - This is INFO 3 - {{{{x,3}}{{y,foo}}}}
190DEBUG root testPattern (test_log.py:{0[5]}) test_log.py({0[5]}) - This is DEBUG 3 - {{{{x,3}}{{y,foo}}}}
191INFO root testPattern (test_log.py:{0[6]}) test_log.py({0[6]}) - This is INFO 4 - {{{{y,foo}}}}
192DEBUG root testPattern (test_log.py:{0[7]}) test_log.py({0[7]}) - This is DEBUG 4 - {{{{y,foo}}}}
193INFO root testPattern (test_log.py:{0[8]}) test_log.py({0[8]}) - This is INFO 5 - {{{{y,foo}}}}
194DEBUG root testPattern (test_log.py:{0[9]}) test_log.py({0[9]}) - This is DEBUG 5 - {{{{y,foo}}}}
195""".format([x + 157 for x in (0, 1, 8, 9, 13, 14, 17, 18, 21, 22)], __name__)) # noqa E501 line too long
197 def testMDCPutPid(self):
198 """
199 Test add of PID Mapped Diagnostic Context (MDC).
200 """
201 pid = os.fork()
202 try:
204 log.MDC("PID", os.getpid())
205 self.configure("""
206log4j.rootLogger=DEBUG, CA
207log4j.appender.CA=ConsoleAppender
208log4j.appender.CA.layout=PatternLayout
209log4j.appender.CA.layout.ConversionPattern=%-5p PID:%X{{PID}} %c %C %M (%F:%L) %l - %m%n
210""") # noqa E501 line too long
211 self.assertGreaterEqual(pid, 0, "Failed to fork")
213 msg = "This is INFO"
214 if pid == 0:
215 self.tempDir = tempfile.mkdtemp()
216 self.outputFilename = os.path.join(self.tempDir,
217 "log-child.out")
218 msg += " in child process"
219 elif pid > 0:
220 child_pid, child_status = os.wait()
221 self.assertEqual(child_status, 0,
222 "Child returns incorrect code")
223 msg += " in parent process"
225 with TestLog.StdoutCapture(self.outputFilename):
226 log.info(msg)
227 line = 226 # line number for previous line
228 finally:
229 log.MDCRemove("PID")
231 # Use format to make line numbers easier to change.
232 self.check("""
233INFO PID:{1} root testMDCPutPid (test_log.py:{0}) test_log.py({0}) - {2}
234""".format(line, os.getpid(), msg))
236 # don't pass other tests in child process
237 if pid == 0:
238 os._exit(0)
240 def testFileAppender(self):
241 """Test configuring logging to go to a file."""
242 self.configure("""
243log4j.rootLogger=DEBUG, FA
244log4j.appender.FA=FileAppender
245log4j.appender.FA.file={0}
246log4j.appender.FA.layout=SimpleLayout
247""")
248 log.MDC("x", 3)
249 log.trace("This is TRACE")
250 log.info("This is INFO")
251 log.debug("This is DEBUG")
252 log.MDCRemove("x")
254 self.check("""
255INFO - This is INFO
256DEBUG - This is DEBUG
257""")
259 def testPythonLogging(self):
260 """Test logging through the Python logging interface."""
261 with TestLog.StdoutCapture(self.outputFilename):
262 lgr = logging.getLogger()
263 lgr.setLevel(logging.INFO)
264 log.configure()
265 with self.assertLogs(level="INFO") as cm:
266 # Force the lsst.log handler to be applied as well as the
267 # unittest log handler
268 lgr.addHandler(log.LogHandler())
269 lgr.info("This is INFO")
270 lgr.debug("This is DEBUG")
271 lgr.warning("This is %s", "WARNING")
272 # message can be arbitrary Python object
273 lgr.info(((1, 2), (3, 4)))
274 lgr.info({1: 2})
276 # Confirm that Python logging also worked
277 self.assertEqual(len(cm.output), 4, f"Got output: {cm.output}")
278 logging.shutdown()
280 self.check("""
281root INFO: This is INFO
282root WARN: This is WARNING
283root INFO: ((1, 2), (3, 4))
284root INFO: {1: 2}
285""")
287 def testMdcInit(self):
289 expected_msg = \
290 "INFO - main thread {{MDC_INIT,OK}}\n" + \
291 "INFO - thread 1 {{MDC_INIT,OK}}\n" + \
292 "INFO - thread 2 {{MDC_INIT,OK}}\n"
294 with TestLog.StdoutCapture(self.outputFilename):
296 self.configure("""
297log4j.rootLogger=DEBUG, CA
298log4j.appender.CA=ConsoleAppender
299log4j.appender.CA.layout=PatternLayout
300log4j.appender.CA.layout.ConversionPattern=%-5p - %m %X%n
301""")
303 def fun():
304 log.MDC("MDC_INIT", "OK")
305 log.MDCRegisterInit(fun)
307 log.info("main thread")
309 thread = threading.Thread(target=lambda: log.info("thread 1"))
310 thread.start()
311 thread.join()
313 thread = threading.Thread(target=lambda: log.info("thread 2"))
314 thread.start()
315 thread.join()
317 self.check(expected_msg)
319 log.MDCRemove("MDC_INIT")
321 def testMdcUpdate(self):
322 """Test for overwriting MDC.
323 """
325 expected_msg = \
326 "INFO - Message one {}\n" \
327 "INFO - Message two {{LABEL,123456}}\n" \
328 "INFO - Message three {{LABEL,654321}}\n" \
329 "INFO - Message four {}\n"
331 with TestLog.StdoutCapture(self.outputFilename):
333 self.configure("""
334log4j.rootLogger=DEBUG, CA
335log4j.appender.CA=ConsoleAppender
336log4j.appender.CA.layout=PatternLayout
337log4j.appender.CA.layout.ConversionPattern=%-5p - %m %X%n
338""")
340 log.info("Message one")
342 log.MDC("LABEL", "123456")
343 log.info("Message two")
345 log.MDC("LABEL", "654321")
346 log.info("Message three")
348 log.MDCRemove("LABEL")
349 log.info("Message four")
351 self.check(expected_msg)
353 def testLwpID(self):
354 """Test log.lwpID() method."""
355 lwp1 = log.lwpID()
356 lwp2 = log.lwpID()
358 self.assertEqual(lwp1, lwp2)
360 def testLogger(self):
361 """
362 Test log object.
363 """
364 with TestLog.StdoutCapture(self.outputFilename):
365 log.configure()
366 logger = log.Log.getLogger("b")
367 self.assertEqual(logger.getName(), "b")
368 logger.trace("This is TRACE")
369 logger.info("This is INFO")
370 logger.debug("This is DEBUG")
371 logger.warn("This is WARN")
372 logger.error("This is ERROR")
373 logger.fatal("This is FATAL")
374 logger.warn("Format %d %g %s", 3, 2.71828, "foo")
375 self.check("""
376b INFO: This is INFO
377b WARN: This is WARN
378b ERROR: This is ERROR
379b FATAL: This is FATAL
380b WARN: Format 3 2.71828 foo
381""")
383 def testLoggerLevel(self):
384 """
385 Test levels of Log objects
386 """
387 with TestLog.StdoutCapture(self.outputFilename):
388 self.configure("""
389log4j.rootLogger=TRACE, CA
390log4j.appender.CA=ConsoleAppender
391log4j.appender.CA.layout=PatternLayout
392log4j.appender.CA.layout.ConversionPattern=%-5p %c (%F)- %m%n
393""")
394 self.assertEqual(log.Log.getLevel(log.Log.getDefaultLogger()),
395 log.TRACE)
396 logger = log.Log.getLogger("a.b")
397 self.assertEqual(logger.getName(), "a.b")
398 logger.trace("This is TRACE")
399 logger.setLevel(log.INFO)
400 self.assertEqual(logger.getLevel(), log.INFO)
401 self.assertEqual(log.Log.getLevel(logger), log.INFO)
402 logger.debug("This is DEBUG")
403 logger.info("This is INFO")
404 logger.fatal("Format %d %g %s", 3, 2.71828, "foo")
406 logger = log.Log.getLogger("a.b.c")
407 self.assertEqual(logger.getName(), "a.b.c")
408 logger.trace("This is TRACE")
409 logger.debug("This is DEBUG")
410 logger.warn("This is WARN")
411 logger.error("This is ERROR")
412 logger.fatal("This is FATAL")
413 logger.info("Format %d %g %s", 3, 2.71828, "foo")
414 self.check("""
415TRACE a.b (test_log.py)- This is TRACE
416INFO a.b (test_log.py)- This is INFO
417FATAL a.b (test_log.py)- Format 3 2.71828 foo
418WARN a.b.c (test_log.py)- This is WARN
419ERROR a.b.c (test_log.py)- This is ERROR
420FATAL a.b.c (test_log.py)- This is FATAL
421INFO a.b.c (test_log.py)- Format 3 2.71828 foo
422""")
424 def testMsgWithPercentS(self):
425 """Test logging messages containing %s (DM-7509)
426 """
427 with TestLog.StdoutCapture(self.outputFilename):
428 log.configure()
429 logger = log.Log()
430 logger.info("INFO with %s")
431 logger.trace("TRACE with %s")
432 logger.debug("DEBUG with %s")
433 logger.warn("WARN with %s")
434 logger.error("ERROR with %s")
435 logger.fatal("FATAL with %s")
436 logger.logMsg(log.DEBUG, "foo", "bar", 5, "DEBUG with %s")
437 self.check("""
438root INFO: INFO with %s
439root WARN: WARN with %s
440root ERROR: ERROR with %s
441root FATAL: FATAL with %s
442root DEBUG: DEBUG with %s
443""")
445 def testForwardToPython(self):
446 """Test that `lsst.log` log messages can be forwarded to `logging`."""
447 log.configure()
449 # Without forwarding we only get python logger messages captured
450 with self.assertLogs(level="WARNING") as cm:
451 log.warn("lsst.log warning message that will not be forwarded to Python")
452 logging.warning("Python logging message that will be captured")
453 self.assertEqual(len(cm.output), 1)
455 log.usePythonLogging()
457 # With forwarding we get 2 logging messages captured
458 with self.assertLogs(level="WARNING") as cm:
459 log.warn("This is a warning from lsst log meant for python logging")
460 logging.warning("Python warning log message to be captured")
461 self.assertEqual(len(cm.output), 2)
463 loggername = "newlogger"
464 log2 = log.Log.getLogger(loggername)
465 with self.assertLogs(level="INFO", logger=loggername):
466 log2.info("Info message to non-root lsst logger")
468 # Check that debug and info are working properly
469 # This test should return a single log message
470 with self.assertLogs(level="INFO", logger=loggername) as cm:
471 log2.info("Second INFO message to non-root lsst logger")
472 log.debug("Debug message to root lsst logger")
474 self.assertEqual(len(cm.output), 1, f"Got output: {cm.output}")
476 logging.shutdown()
478 def testLogLoop(self):
479 """Test that Python log forwarding works even if Python logging has
480 been forwarded to lsst.log"""
482 log.configure()
484 # Note that assertLogs causes a specialists Python logging handler
485 # to be added.
487 # Set up some Python loggers
488 loggername = "testLogLoop"
489 lgr = logging.getLogger(loggername)
490 lgr.setLevel(logging.INFO)
491 rootlgr = logging.getLogger()
492 rootlgr.setLevel(logging.INFO)
494 # Declare that we are using the Python logger and that this will
495 # not cause a log loop if we also are forwarding Python logging to
496 # lsst.log
497 log.usePythonLogging()
499 # Ensure that we can log both in lsst.log and Python
500 rootlgr.addHandler(log.LogHandler())
502 # All three of these messages go through LogHandler
503 # The first two because they have the handler added explicitly, the
504 # the final one because the lsst.log logger is forwarded to the
505 # ROOT Python logger which has the LogHandler registered.
507 with open(self.outputFilename, "w") as fd:
508 # Adding a StreamHandler will cause the LogHandler to no-op
509 streamHandler = logging.StreamHandler(stream=fd)
510 rootlgr.addHandler(streamHandler)
512 # Do not use assertLogs since that messes with handlers
513 lgr.info("INFO message: Python child logger, lsst.log.LogHandler + PythonLogging")
514 rootlgr.info("INFO message: Python root logger, lsst.log.logHandler + PythonLogging")
516 # This will use a ROOT python logger which has a LogHandler attached
517 log.info("INFO message: lsst.log root logger, PythonLogging")
519 rootlgr.removeHandler(streamHandler)
521 self.check("""
522INFO message: Python child logger, lsst.log.LogHandler + PythonLogging
523INFO message: Python root logger, lsst.log.logHandler + PythonLogging
524INFO message: lsst.log root logger, PythonLogging""")
526 with open(self.outputFilename, "w") as fd:
527 # Adding a StreamHandler will cause the LogHandler to no-op
528 streamHandler = logging.StreamHandler(stream=fd)
529 rootlgr.addHandler(streamHandler)
531 # Do not use assertLogs since that messes with handlers
532 lgr.info("INFO message: Python child logger, lsst.log.LogHandler + PythonLogging")
533 rootlgr.info("INFO message: Python root logger, lsst.log.logHandler + PythonLogging")
535 # This will use a ROOT python logger which has a LogHandler attached
536 log.info("INFO message: lsst.log root logger, PythonLogging")
538 rootlgr.removeHandler(streamHandler)
540 self.check("""
541INFO message: Python child logger, lsst.log.LogHandler + PythonLogging
542INFO message: Python root logger, lsst.log.logHandler + PythonLogging
543INFO message: lsst.log root logger, PythonLogging""")
545 with self.assertLogs(level="INFO") as cm:
546 rootlgr.info("Python log message forward to lsst.log")
547 log.info("lsst.log message forwarded to Python")
549 self.assertEqual(len(cm.output), 2, f"Got output: {cm.output}")
551 logging.shutdown()
553 def testForwardToPythonContextManager(self):
554 """Test that `lsst.log` log messages can be forwarded to `logging`
555 using context manager"""
556 log.configure()
558 # Without forwarding we only get python logger messages captured
559 with self.assertLogs(level="WARNING") as cm:
560 log.warning("lsst.log: not forwarded")
561 logging.warning("Python logging: captured")
562 self.assertEqual(len(cm.output), 1)
564 # Temporarily turn on forwarding
565 with log.UsePythonLogging():
566 with self.assertLogs(level="WARNING") as cm:
567 log.warn("lsst.log: forwarded")
568 logging.warning("Python logging: also captured")
569 self.assertEqual(len(cm.output), 2)
571 # Verify that forwarding is disabled
572 self.assertFalse(log.Log.UsePythonLogging)
574 def testLevelTranslator(self):
575 """Test LevelTranslator class
576 """
577 # correspondence between levels, logging has no TRACE but we accept
578 # small integer in its place
579 levelMap = ((log.TRACE, 5),
580 (log.DEBUG, logging.DEBUG),
581 (log.INFO, logging.INFO),
582 (log.WARN, logging.WARNING),
583 (log.ERROR, logging.ERROR),
584 (log.FATAL, logging.FATAL))
585 for logLevel, loggingLevel in levelMap:
586 self.assertEqual(log.LevelTranslator.lsstLog2logging(logLevel), loggingLevel)
587 self.assertEqual(log.LevelTranslator.logging2lsstLog(loggingLevel), logLevel)
589 def testChildLogger(self):
590 """Check the getChild logger method."""
591 logger = log.getDefaultLogger()
592 self.assertEqual(logger.getName(), "")
593 logger1 = logger.getChild("child1")
594 self.assertEqual(logger1.getName(), "child1")
595 logger2 = logger1.getChild("child2")
596 self.assertEqual(logger2.getName(), "child1.child2")
597 logger2a = logger1.getChild(".child2")
598 self.assertEqual(logger2a.getName(), "child1.child2")
599 logger3 = logger2.getChild(" .. child3")
600 self.assertEqual(logger3.getName(), "child1.child2.child3")
601 logger3a = logger1.getChild("child2.child3")
602 self.assertEqual(logger3a.getName(), "child1.child2.child3")
605if __name__ == "__main__": 605 ↛ 606line 605 didn't jump to line 606, because the condition on line 605 was never true
606 unittest.main()