In this tutorial I am going to illustrate how you can set up logs to enable logging in Python
Prerequisites: Python
Step 1:
Install python-logstash-async (which is as of Jan 2023 is the library that works best and has a lot of customization options)
pip install python-logstash-async
Step 2:
Create a file called logger.py in your application root folder and populate it with the below code. Replace ‘myapp’ with your application name.
Also replace the values of logstash-remote-url(String) and logstash-port(Int) values in the code with your logstash url and port.
import logging
import sys
from logging.handlers import TimedRotatingFileHandler
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.handler import LogstashFormatter
from logstash_async.constants import constants
FORMATTER = logging.Formatter("%(asctime)s-%(name)s-%(levelname)s-%(message)s")
LOG_FILE = "myapp.log"
def get_logstash_handler():
'''
Handler for writing logs to logstash
'''
constants.FORMATTER_RECORD_FIELD_SKIP_LIST = [
'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
'funcName', 'id', 'levelname', 'levelno', 'lineno', 'module',
'msecs', 'msg', 'name', 'pathname', 'process',
'processName', 'relativeCreated', 'stack_info', 'pid',
'interpreter_version', 'program', 'process_name', 'interpreter',
'port', 'logsource', 'logstash_async_version', 'path', 'thread_name',
'host', 'thread', 'threadName', '@version']
constants.FORMATTER_LOGSTASH_MESSAGE_FIELD_LIST = [
'@timestamp', 'level', 'logsource', 'message', 'program', 'type', 'tags', '@metadata']
logstash_formatter = LogstashFormatter(extra_prefix=None,
extra=dict(application='myapp'))
logstash_handler = AsynchronousLogstashHandler(
host=<logstash-remote-url>,
port=<logstash-port>,
database_path=None,
)
logstash_handler.setFormatter(logstash_formatter)
return logstash_handler
def get_console_handler():
'''
Handler for writing logs to stdout
'''
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(FORMATTER)
return console_handler
def get_file_handler():
'''
Handler for writing logs to file
'''
file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight')
file_handler.setFormatter(FORMATTER)
return file_handler
def get_logger(logger_name):
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG) # better to have too much log than not enough
logger.addHandler(get_console_handler())
logger.addHandler(get_file_handler())
logger.addHandler(get_logstash_handler())
# with this pattern, it's rarely necessary to propagate the error up to parent
logger.propagate = False
return logger
Step 3:
You may use the logger in your application code as below
from logger import get_logger
log = get_logger(__name__)
log.debug("this is a debug log")
log.info("this is an info")
log.warning("this is a warning")
log.error("this is an error")
log.exception("this is an exception")