Making Python loggers output all messages to stdout in addition to log file
Question:
Is there a way to make Python logging using the logging
module automatically output things to stdout in addition to the log file where they are supposed to go? For example, I’d like all calls to logger.warning
, logger.critical
, logger.error
to go to their intended places but in addition always be copied to stdout
. This is to avoid duplicating messages like:
mylogger.critical("something failed")
print("something failed")
Answers:
All logging output is handled by the handlers; just add a logging.StreamHandler()
to the root logger.
Here’s an example configuring a stream handler (using stdout
instead of the default stderr
) and adding it to the root logger:
import logging
import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
root.addHandler(handler)
It’s possible using multiple handlers.
import logging
import auxiliary_module
# create logger with 'spam_application'
log = logging.getLogger('spam_application')
log.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# create file handler which logs even debug messages
fh = logging.FileHandler('spam.log')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
log.addHandler(fh)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
ch.setFormatter(formatter)
log.addHandler(ch)
log.info('creating an instance of auxiliary_module.Auxiliary')
a = auxiliary_module.Auxiliary()
log.info('created an instance of auxiliary_module.Auxiliary')
log.info('calling auxiliary_module.Auxiliary.do_something')
a.do_something()
log.info('finished auxiliary_module.Auxiliary.do_something')
log.info('calling auxiliary_module.some_function()')
auxiliary_module.some_function()
log.info('done with auxiliary_module.some_function()')
# remember to close the handlers
for handler in log.handlers:
handler.close()
log.removeFilter(handler)
Please see: https://docs.python.org/2/howto/logging-cookbook.html
The simplest way to log to stdout using basicConfig
:
import logging
import sys
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
The simplest way to log to file and to stderr:
import logging
logging.basicConfig(filename="logfile.txt")
stderrLogger=logging.StreamHandler()
stderrLogger.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logging.getLogger().addHandler(stderrLogger)
You could create two handlers for file and stdout and then create one logger with handlers
argument to basicConfig
. It could be useful if you have the same log_level and format output for both handlers:
import logging
import sys
file_handler = logging.FileHandler(filename='tmp.log')
stdout_handler = logging.StreamHandler(stream=sys.stdout)
handlers = [file_handler, stdout_handler]
logging.basicConfig(
level=logging.DEBUG,
format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
handlers=handlers
)
logger = logging.getLogger('LOGGER_NAME')
Here is a solution based on the powerful but poorly documented logging.config.dictConfig
method.
Instead of sending every log message to stdout
, it sends messages with log level ERROR
and higher to stderr
and everything else to stdout
.
This can be useful if other parts of the system are listening to stderr
or stdout
.
import logging
import logging.config
import sys
class _ExcludeErrorsFilter(logging.Filter):
def filter(self, record):
"""Only lets through log messages with log level below ERROR ."""
return record.levelno < logging.ERROR
config = {
'version': 1,
'filters': {
'exclude_errors': {
'()': _ExcludeErrorsFilter
}
},
'formatters': {
# Modify log message format here or replace with your custom formatter class
'my_formatter': {
'format': '(%(process)d) %(asctime)s %(name)s (line %(lineno)s) | %(levelname)s %(message)s'
}
},
'handlers': {
'console_stderr': {
# Sends log messages with log level ERROR or higher to stderr
'class': 'logging.StreamHandler',
'level': 'ERROR',
'formatter': 'my_formatter',
'stream': sys.stderr
},
'console_stdout': {
# Sends log messages with log level lower than ERROR to stdout
'class': 'logging.StreamHandler',
'level': 'DEBUG',
'formatter': 'my_formatter',
'filters': ['exclude_errors'],
'stream': sys.stdout
},
'file': {
# Sends all log messages to a file
'class': 'logging.FileHandler',
'level': 'DEBUG',
'formatter': 'my_formatter',
'filename': 'my.log',
'encoding': 'utf8'
}
},
'root': {
# In general, this should be kept at 'NOTSET'.
# Otherwise it would interfere with the log levels set for each handler.
'level': 'NOTSET',
'handlers': ['console_stderr', 'console_stdout', 'file']
},
}
logging.config.dictConfig(config)
Since no one has shared a neat two liner, I will share my own:
logging.basicConfig(filename='logs.log', level=logging.DEBUG, format="%(asctime)s:%(levelname)s: %(message)s")
logging.getLogger().addHandler(logging.StreamHandler())
Here’s an extremely simple example:
import logging
l = logging.getLogger("test")
# Add a file logger
f = logging.FileHandler("test.log")
l.addHandler(f)
# Add a stream logger
s = logging.StreamHandler()
l.addHandler(s)
# Send a test message to both -- critical will always log
l.critical("test msg")
The output will show “test msg” on stdout and also in the file.
I simplified my source code (whose original version is OOP and uses a configuration file), to give you an alternative solution to @EliasStrehle’s one, without using the dictConfig (thus easiest to integrate with existing source code):
import logging
import sys
def create_stream_handler(stream, formatter, level, message_filter=None):
handler = logging.StreamHandler(stream=stream)
handler.setLevel(level)
handler.setFormatter(formatter)
if message_filter:
handler.addFilter(message_filter)
return handler
def configure_logger(logger: logging.Logger, enable_console: bool = True, enable_file: bool = True):
if not logger.handlers:
if enable_console:
message_format: str = '{asctime:20} {name:16} {levelname:8} {message}'
date_format: str = '%Y/%m/%d %H:%M:%S'
level: int = logging.DEBUG
formatter = logging.Formatter(message_format, date_format, '{')
# Configures error output (from Warning levels).
error_output_handler = create_stream_handler(sys.stderr, formatter,
max(level, logging.WARNING))
logger.addHandler(error_output_handler)
# Configures standard output (from configured Level, if lower than Warning,
# and excluding everything from Warning and higher).
if level < logging.WARNING:
standard_output_filter = lambda record: record.levelno < logging.WARNING
standard_output_handler = create_stream_handler(sys.stdout, formatter, level,
standard_output_filter)
logger.addHandler(standard_output_handler)
if enable_file:
message_format: str = '{asctime:20} {name:16} {levelname:8} {message}'
date_format: str = '%Y/%m/%d %H:%M:%S'
level: int = logging.DEBUG
output_file: str = '/tmp/so_test.log'
handler = logging.FileHandler(output_file)
formatter = logging.Formatter(message_format, date_format, '{')
handler.setLevel(level)
handler.setFormatter(formatter)
logger.addHandler(handler)
This is a very simple way to test it:
logger: logging.Logger = logging.getLogger('MyLogger')
logger.setLevel(logging.DEBUG)
configure_logger(logger, True, True)
logger.debug('Debug message ...')
logger.info('Info message ...')
logger.warning('Warning ...')
logger.error('Error ...')
logger.fatal('Fatal message ...')
For more detailed explanations – great documentation at that link.
For example: It’s easy, you only need to set up two loggers.
import sys
import logging
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('my_log_info.log')
sh = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S')
fh.setFormatter(formatter)
sh.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(sh)
def hello_logger():
logger.info("Hello info")
logger.critical("Hello critical")
logger.warning("Hello warning")
logger.debug("Hello debug")
if __name__ == "__main__":
print(hello_logger())
Output – terminal:
[Mon, 10 Aug 2020 12:44:25] INFO [TestLoger.py.hello_logger:15] Hello info
[Mon, 10 Aug 2020 12:44:25] CRITICAL [TestLoger.py.hello_logger:16] Hello critical
[Mon, 10 Aug 2020 12:44:25] WARNING [TestLoger.py.hello_logger:17] Hello warning
[Mon, 10 Aug 2020 12:44:25] DEBUG [TestLoger.py.hello_logger:18] Hello debug
None
Output – in file:
UPDATE: color terminal
Package:
pip install colorlog
Code:
import sys
import logging
import colorlog
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('my_log_info.log')
sh = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S')
fh.setFormatter(formatter)
sh.setFormatter(colorlog.ColoredFormatter('%(log_color)s [%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S'))
logger.addHandler(fh)
logger.addHandler(sh)
def hello_logger():
logger.info("Hello info")
logger.critical("Hello critical")
logger.warning("Hello warning")
logger.debug("Hello debug")
logger.error("Error message")
if __name__ == "__main__":
hello_logger()
Recommendation:
Complete logger configuration from INI
file, which also includes setup for stdout
and debug.log
:
handler_file
level=WARNING
handler_screen
level=DEBUG
You should use tee
. Configure the app to write to stdout, and run
python3 app.py | tee log.txt
Then you’ll get the log messages in your stdout as well as in log.txt.
Is there a way to make Python logging using the logging
module automatically output things to stdout in addition to the log file where they are supposed to go? For example, I’d like all calls to logger.warning
, logger.critical
, logger.error
to go to their intended places but in addition always be copied to stdout
. This is to avoid duplicating messages like:
mylogger.critical("something failed")
print("something failed")
All logging output is handled by the handlers; just add a logging.StreamHandler()
to the root logger.
Here’s an example configuring a stream handler (using stdout
instead of the default stderr
) and adding it to the root logger:
import logging
import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
root.addHandler(handler)
It’s possible using multiple handlers.
import logging
import auxiliary_module
# create logger with 'spam_application'
log = logging.getLogger('spam_application')
log.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# create file handler which logs even debug messages
fh = logging.FileHandler('spam.log')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
log.addHandler(fh)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
ch.setFormatter(formatter)
log.addHandler(ch)
log.info('creating an instance of auxiliary_module.Auxiliary')
a = auxiliary_module.Auxiliary()
log.info('created an instance of auxiliary_module.Auxiliary')
log.info('calling auxiliary_module.Auxiliary.do_something')
a.do_something()
log.info('finished auxiliary_module.Auxiliary.do_something')
log.info('calling auxiliary_module.some_function()')
auxiliary_module.some_function()
log.info('done with auxiliary_module.some_function()')
# remember to close the handlers
for handler in log.handlers:
handler.close()
log.removeFilter(handler)
Please see: https://docs.python.org/2/howto/logging-cookbook.html
The simplest way to log to stdout using basicConfig
:
import logging
import sys
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
The simplest way to log to file and to stderr:
import logging
logging.basicConfig(filename="logfile.txt")
stderrLogger=logging.StreamHandler()
stderrLogger.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logging.getLogger().addHandler(stderrLogger)
You could create two handlers for file and stdout and then create one logger with handlers
argument to basicConfig
. It could be useful if you have the same log_level and format output for both handlers:
import logging
import sys
file_handler = logging.FileHandler(filename='tmp.log')
stdout_handler = logging.StreamHandler(stream=sys.stdout)
handlers = [file_handler, stdout_handler]
logging.basicConfig(
level=logging.DEBUG,
format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
handlers=handlers
)
logger = logging.getLogger('LOGGER_NAME')
Here is a solution based on the powerful but poorly documented logging.config.dictConfig
method.
Instead of sending every log message to stdout
, it sends messages with log level ERROR
and higher to stderr
and everything else to stdout
.
This can be useful if other parts of the system are listening to stderr
or stdout
.
import logging
import logging.config
import sys
class _ExcludeErrorsFilter(logging.Filter):
def filter(self, record):
"""Only lets through log messages with log level below ERROR ."""
return record.levelno < logging.ERROR
config = {
'version': 1,
'filters': {
'exclude_errors': {
'()': _ExcludeErrorsFilter
}
},
'formatters': {
# Modify log message format here or replace with your custom formatter class
'my_formatter': {
'format': '(%(process)d) %(asctime)s %(name)s (line %(lineno)s) | %(levelname)s %(message)s'
}
},
'handlers': {
'console_stderr': {
# Sends log messages with log level ERROR or higher to stderr
'class': 'logging.StreamHandler',
'level': 'ERROR',
'formatter': 'my_formatter',
'stream': sys.stderr
},
'console_stdout': {
# Sends log messages with log level lower than ERROR to stdout
'class': 'logging.StreamHandler',
'level': 'DEBUG',
'formatter': 'my_formatter',
'filters': ['exclude_errors'],
'stream': sys.stdout
},
'file': {
# Sends all log messages to a file
'class': 'logging.FileHandler',
'level': 'DEBUG',
'formatter': 'my_formatter',
'filename': 'my.log',
'encoding': 'utf8'
}
},
'root': {
# In general, this should be kept at 'NOTSET'.
# Otherwise it would interfere with the log levels set for each handler.
'level': 'NOTSET',
'handlers': ['console_stderr', 'console_stdout', 'file']
},
}
logging.config.dictConfig(config)
Since no one has shared a neat two liner, I will share my own:
logging.basicConfig(filename='logs.log', level=logging.DEBUG, format="%(asctime)s:%(levelname)s: %(message)s")
logging.getLogger().addHandler(logging.StreamHandler())
Here’s an extremely simple example:
import logging
l = logging.getLogger("test")
# Add a file logger
f = logging.FileHandler("test.log")
l.addHandler(f)
# Add a stream logger
s = logging.StreamHandler()
l.addHandler(s)
# Send a test message to both -- critical will always log
l.critical("test msg")
The output will show “test msg” on stdout and also in the file.
I simplified my source code (whose original version is OOP and uses a configuration file), to give you an alternative solution to @EliasStrehle’s one, without using the dictConfig (thus easiest to integrate with existing source code):
import logging
import sys
def create_stream_handler(stream, formatter, level, message_filter=None):
handler = logging.StreamHandler(stream=stream)
handler.setLevel(level)
handler.setFormatter(formatter)
if message_filter:
handler.addFilter(message_filter)
return handler
def configure_logger(logger: logging.Logger, enable_console: bool = True, enable_file: bool = True):
if not logger.handlers:
if enable_console:
message_format: str = '{asctime:20} {name:16} {levelname:8} {message}'
date_format: str = '%Y/%m/%d %H:%M:%S'
level: int = logging.DEBUG
formatter = logging.Formatter(message_format, date_format, '{')
# Configures error output (from Warning levels).
error_output_handler = create_stream_handler(sys.stderr, formatter,
max(level, logging.WARNING))
logger.addHandler(error_output_handler)
# Configures standard output (from configured Level, if lower than Warning,
# and excluding everything from Warning and higher).
if level < logging.WARNING:
standard_output_filter = lambda record: record.levelno < logging.WARNING
standard_output_handler = create_stream_handler(sys.stdout, formatter, level,
standard_output_filter)
logger.addHandler(standard_output_handler)
if enable_file:
message_format: str = '{asctime:20} {name:16} {levelname:8} {message}'
date_format: str = '%Y/%m/%d %H:%M:%S'
level: int = logging.DEBUG
output_file: str = '/tmp/so_test.log'
handler = logging.FileHandler(output_file)
formatter = logging.Formatter(message_format, date_format, '{')
handler.setLevel(level)
handler.setFormatter(formatter)
logger.addHandler(handler)
This is a very simple way to test it:
logger: logging.Logger = logging.getLogger('MyLogger')
logger.setLevel(logging.DEBUG)
configure_logger(logger, True, True)
logger.debug('Debug message ...')
logger.info('Info message ...')
logger.warning('Warning ...')
logger.error('Error ...')
logger.fatal('Fatal message ...')
For more detailed explanations – great documentation at that link.
For example: It’s easy, you only need to set up two loggers.
import sys
import logging
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('my_log_info.log')
sh = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S')
fh.setFormatter(formatter)
sh.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(sh)
def hello_logger():
logger.info("Hello info")
logger.critical("Hello critical")
logger.warning("Hello warning")
logger.debug("Hello debug")
if __name__ == "__main__":
print(hello_logger())
Output – terminal:
[Mon, 10 Aug 2020 12:44:25] INFO [TestLoger.py.hello_logger:15] Hello info
[Mon, 10 Aug 2020 12:44:25] CRITICAL [TestLoger.py.hello_logger:16] Hello critical
[Mon, 10 Aug 2020 12:44:25] WARNING [TestLoger.py.hello_logger:17] Hello warning
[Mon, 10 Aug 2020 12:44:25] DEBUG [TestLoger.py.hello_logger:18] Hello debug
None
Output – in file:
UPDATE: color terminal
Package:
pip install colorlog
Code:
import sys
import logging
import colorlog
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('my_log_info.log')
sh = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S')
fh.setFormatter(formatter)
sh.setFormatter(colorlog.ColoredFormatter('%(log_color)s [%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S'))
logger.addHandler(fh)
logger.addHandler(sh)
def hello_logger():
logger.info("Hello info")
logger.critical("Hello critical")
logger.warning("Hello warning")
logger.debug("Hello debug")
logger.error("Error message")
if __name__ == "__main__":
hello_logger()
Recommendation:
Complete logger configuration from INI
file, which also includes setup for stdout
and debug.log
:
handler_file
level=WARNING
handler_screen
level=DEBUG
You should use tee
. Configure the app to write to stdout, and run
python3 app.py | tee log.txt
Then you’ll get the log messages in your stdout as well as in log.txt.