eveAI/config/logging_config.py

import os
from graypy import GELFUDPHandler
import logging
import logging.config

# Graylog configuration
GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost')
GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201))
env = os.environ.get('FLASK_ENV', 'development')


class CustomLogRecord(logging.LogRecord):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')

    def __setattr__(self, name, value):
        if name not in {'event_type', 'tenant_id', 'trace_id', 'span_id', 'span_name', 'parent_span_id',
                        'document_version_id', 'chat_session_id', 'interaction_id', 'environment'}:
            super().__setattr__(name, value)


def custom_log_record_factory(*args, **kwargs):
    record = CustomLogRecord(*args, **kwargs)
    return record


# Set the custom log record factory
logging.setLogRecordFactory(custom_log_record_factory)


LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'file_app': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/eveai_app.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_workers': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/eveai_workers.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_chat': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/eveai_chat.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_chat_workers': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/eveai_chat_workers.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_api': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/eveai_api.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_sqlalchemy': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/sqlalchemy.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_mailman': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/mailman.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_security': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/security.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_rag_tuning': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/rag_tuning.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_embed_tuning': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/embed_tuning.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'file_business_events': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/business_events.log',
            'maxBytes': 1024 * 1024 * 5,  # 5MB
            'backupCount': 10,
            'formatter': 'standard',
        },
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'DEBUG',
            'formatter': 'standard',
        },
        'graylog': {
            'level': 'DEBUG',
            'class': 'graypy.GELFUDPHandler',
            'host': GRAYLOG_HOST,
            'port': GRAYLOG_PORT,
            'debugging_fields': True,  # Set to True if you want to include debugging fields
            'extra_fields': True,  # Set to True if you want to include extra fields
        },
    },
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
                      '[Thread: %(threadName)s]: %(message)s'
        },
        'graylog': {
            'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
                      '[Thread: %(threadName)s]: %(message)s',
            'datefmt': '%Y-%m-%d %H:%M:%S',
        },
    },
    'loggers': {
        'eveai_app': {  # logger for the eveai_app
            'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'eveai_workers': {  # logger for the eveai_workers
            'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'eveai_chat': {  # logger for the eveai_chat
            'handlers': ['file_chat', 'graylog', ] if env == 'production' else ['file_chat', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'eveai_chat_workers': {  # logger for the eveai_chat_workers
            'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'eveai_api': {  # logger for the eveai_chat_workers
            'handlers': ['file_api', 'graylog', ] if env == 'production' else ['file_api', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'sqlalchemy.engine': {  # logger for the sqlalchemy
            'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'mailman': {  # logger for the mailman
            'handlers': ['file_mailman', 'graylog', ] if env == 'production' else ['file_mailman', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'security': {  # logger for the security
            'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'rag_tuning': {  # logger for the rag_tuning
            'handlers': ['file_rag_tuning', 'graylog', ] if env == 'production' else ['file_rag_tuning', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'embed_tuning': {  # logger for the embed_tuning
            'handlers': ['file_embed_tuning', 'graylog', ] if env == 'production' else ['file_embed_tuning', ],
            'level': 'DEBUG',
            'propagate': False
        },
        'business_events': {
            'handlers': ['file_business_events', 'graylog'],
            'level': 'DEBUG',
            'propagate': False
        },
        '': {  # root logger
            'handlers': ['console'],
            'level': 'WARNING',  # Set higher level for root to minimize noise
            'propagate': False
        },
    }
}