import json import os from datetime import datetime as dt, timezone as tz from flask import current_app from graypy import GELFUDPHandler import logging import logging.config # Graylog configuration GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost') GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201)) env = os.environ.get('FLASK_ENV', 'development') def pad_string(s, target_length=100, pad_char='-'): """ Pads a string with the specified character until it reaches the target length. Args: s: The original string target_length: The desired total length pad_char: Character to use for padding Returns: The padded string """ current_length = len(s) if current_length >= target_length: return s padding_needed = target_length - current_length - 1 return s + " " + (pad_char * padding_needed) class TuningLogRecord(logging.LogRecord): """Extended LogRecord that handles both tuning and business event logging""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Initialize extra fields after parent initialization self._extra_fields = {} self._is_tuning_log = False self._tuning_type = None self._tuning_tenant_id = None self._tuning_catalog_id = None self._tuning_specialist_id = None self._tuning_retriever_id = None self._tuning_processor_id = None self._session_id = None self.component = os.environ.get('COMPONENT_NAME', 'eveai_app') def getMessage(self): """ Override getMessage to handle both string and dict messages """ msg = self.msg if self.args: msg = msg % self.args return msg @property def is_tuning_log(self): return self._is_tuning_log @is_tuning_log.setter def is_tuning_log(self, value): object.__setattr__(self, '_is_tuning_log', value) @property def tuning_type(self): return self._tuning_type @tuning_type.setter def tuning_type(self, value): object.__setattr__(self, '_tuning_type', value) def get_tuning_data(self): """Get all tuning-related data if this is a tuning log""" if not self._is_tuning_log: return {} return { 'is_tuning_log': self._is_tuning_log, 'tuning_type': self._tuning_type, 'tuning_tenant_id': self._tuning_tenant_id, 'tuning_catalog_id': self._tuning_catalog_id, 'tuning_specialist_id': self._tuning_specialist_id, 'tuning_retriever_id': self._tuning_retriever_id, 'tuning_processor_id': self._tuning_processor_id, 'session_id': self._session_id, } def set_tuning_data(self, tenant_id=None, catalog_id=None, specialist_id=None, retriever_id=None, processor_id=None, session_id=None,): """Set tuning-specific data""" object.__setattr__(self, '_tuning_tenant_id', tenant_id) object.__setattr__(self, '_tuning_catalog_id', catalog_id) object.__setattr__(self, '_tuning_specialist_id', specialist_id) object.__setattr__(self, '_tuning_retriever_id', retriever_id) object.__setattr__(self, '_tuning_processor_id', processor_id) object.__setattr__(self, '_session_id', session_id) class TuningFormatter(logging.Formatter): """Universal formatter for all tuning logs""" def __init__(self, fmt=None, datefmt=None): super().__init__(fmt or '%(asctime)s [%(levelname)s] %(name)s: %(message)s', datefmt or '%Y-%m-%d %H:%M:%S') def format(self, record): # First format with the default formatter to handle basic fields formatted_msg = super().format(record) # If this is a tuning log, add the additional context if getattr(record, 'is_tuning_log', False): try: identifiers = [] if hasattr(record, 'tenant_id') and record.tenant_id: identifiers.append(f"Tenant: {record.tenant_id}") if hasattr(record, 'catalog_id') and record.catalog_id: identifiers.append(f"Catalog: {record.catalog_id}") if hasattr(record, 'processor_id') and record.processor_id: identifiers.append(f"Processor: {record.processor_id}") if hasattr(record, 'specialist_id') and record.specialist_id: identifiers.append(f"Specialist: {record.specialist_id}") if hasattr(record, 'retriever_id') and record.retriever_id: identifiers.append(f"Retriever: {record.retriever_id}") if hasattr(record, 'session_id') and record.session_id: identifiers.append(f"Session: {record.session_id}") formatted_msg = ( f"{formatted_msg}\n" f"[TUNING {record.tuning_type}] [{' | '.join(identifiers)}]" ) if hasattr(record, 'tuning_data') and record.tuning_data: formatted_msg += f"\nData: {json.dumps(record.tuning_data, indent=2)}" except Exception as e: return f"{formatted_msg} (Error formatting tuning data: {str(e)})" return formatted_msg class GraylogFormatter(logging.Formatter): """Maintains existing Graylog formatting while adding tuning fields""" def format(self, record): if getattr(record, 'is_tuning_log', False): # Add tuning-specific fields to Graylog record.tuning_fields = { 'is_tuning_log': True, 'tuning_type': record.tuning_type, 'tenant_id': record.tenant_id, 'catalog_id': record.catalog_id, 'specialist_id': record.specialist_id, 'retriever_id': record.retriever_id, 'processor_id': record.processor_id, 'session_id': record.session_id, } return super().format(record) class TuningLogger: """Helper class to manage tuning logs with consistent structure""" def __init__(self, logger_name, tenant_id=None, catalog_id=None, specialist_id=None, retriever_id=None, processor_id=None, session_id=None, log_file=None): """ Initialize a tuning logger Args: logger_name: Base name for the logger tenant_id: Optional tenant ID for context catalog_id: Optional catalog ID for context specialist_id: Optional specialist ID for context retriever_id: Optional retriever ID for context processor_id: Optional processor ID for context session_id: Optional session ID for context and log file naming log_file: Optional custom log file name to use """ self.logger = logging.getLogger(logger_name) self.tenant_id = tenant_id self.catalog_id = catalog_id self.specialist_id = specialist_id self.retriever_id = retriever_id self.processor_id = processor_id self.session_id = session_id self.log_file = log_file # Determine whether to use a session-specific logger if session_id: # Create a unique logger name for this session session_logger_name = f"{logger_name}_{session_id}" self.logger = logging.getLogger(session_logger_name) # If this logger doesn't have handlers yet, configure it if not self.logger.handlers: # Determine log file path if not log_file and session_id: log_file = f"logs/tuning_{session_id}.log" elif not log_file: log_file = "logs/tuning.log" # Configure the logger self._configure_session_logger(log_file) else: # Use the standard tuning logger self.logger = logging.getLogger(logger_name) def _configure_session_logger(self, log_file): """Configure a new session-specific logger with appropriate handlers""" # Create and configure a file handler file_handler = logging.handlers.RotatingFileHandler( filename=log_file, maxBytes=1024 * 1024 * 3, # 3MB backupCount=3 ) file_handler.setFormatter(TuningFormatter()) file_handler.setLevel(logging.DEBUG) # Add the file handler to the logger self.logger.addHandler(file_handler) # Add Graylog handler in production env = os.environ.get('FLASK_ENV', 'development') if env == 'production': try: graylog_handler = GELFUDPHandler( host=GRAYLOG_HOST, port=GRAYLOG_PORT, debugging_fields=True ) graylog_handler.setFormatter(GraylogFormatter()) self.logger.addHandler(graylog_handler) except Exception as e: # Fall back to just file logging if Graylog setup fails fallback_logger = logging.getLogger('eveai_app') fallback_logger.warning(f"Failed to set up Graylog handler: {str(e)}") # Set logger level and disable propagation self.logger.setLevel(logging.DEBUG) self.logger.propagate = False def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DEBUG): """Log a tuning event with structured data""" try: # Create a standard LogRecord for tuning record = logging.LogRecord( name=self.logger.name, level=level, pathname='', lineno=0, msg=pad_string(message, 100, '-'), args=(), exc_info=None ) # Add tuning-specific attributes record.is_tuning_log = True record.tuning_type = tuning_type record.tenant_id = self.tenant_id record.catalog_id = self.catalog_id record.specialist_id = self.specialist_id record.retriever_id = self.retriever_id record.processor_id = self.processor_id record.session_id = self.session_id if data: record.tuning_data = data # Process the record self.logger.handle(record) except Exception as e: fallback_logger = logging.getLogger('eveai_workers') fallback_logger.exception(f"Failed to log tuning message: {str(e)}") # Set the custom log record factory logging.setLogRecordFactory(TuningLogRecord) LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'handlers': { 'file_app': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_app.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_workers': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_workers.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_chat': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_chat.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_chat_workers': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_chat_workers.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_api': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_api.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_beat': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_beat.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_entitlements': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_entitlements.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_sqlalchemy': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/sqlalchemy.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_security': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/security.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_rag_tuning': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/rag_tuning.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_embed_tuning': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/embed_tuning.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'file_business_events': { 'level': 'INFO', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/business_events.log', 'maxBytes': 1024 * 1024 * 1, # 1MB 'backupCount': 2, 'formatter': 'standard', }, 'console': { 'class': 'logging.StreamHandler', 'level': 'DEBUG', 'formatter': 'standard', }, 'tuning_file': { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/tuning.log', 'maxBytes': 1024 * 1024 * 3, # 3MB 'backupCount': 3, 'formatter': 'tuning', }, 'graylog': { 'level': 'DEBUG', 'class': 'graypy.GELFUDPHandler', 'host': GRAYLOG_HOST, 'port': GRAYLOG_PORT, 'debugging_fields': True, 'formatter': 'graylog' }, }, 'formatters': { 'standard': { 'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d]: %(message)s', 'datefmt': '%Y-%m-%d %H:%M:%S' }, 'graylog': { 'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' '[Thread: %(threadName)s]: %(message)s', 'datefmt': '%Y-%m-%d %H:%M:%S', '()': GraylogFormatter }, 'tuning': { '()': TuningFormatter, 'datefmt': '%Y-%m-%d %H:%M:%S UTC' } }, 'loggers': { 'eveai_app': { # logger for the eveai_app 'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_workers': { # logger for the eveai_workers 'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_chat': { # logger for the eveai_chat 'handlers': ['file_chat', 'graylog', ] if env == 'production' else ['file_chat', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_chat_workers': { # logger for the eveai_chat_workers 'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_api': { # logger for the eveai_chat_workers 'handlers': ['file_api', 'graylog', ] if env == 'production' else ['file_api', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_beat': { # logger for the eveai_beat 'handlers': ['file_beat', 'graylog', ] if env == 'production' else ['file_beat', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_entitlements': { # logger for the eveai_entitlements 'handlers': ['file_entitlements', 'graylog', ] if env == 'production' else ['file_entitlements', ], 'level': 'DEBUG', 'propagate': False }, 'sqlalchemy.engine': { # logger for the sqlalchemy 'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ], 'level': 'DEBUG', 'propagate': False }, 'security': { # logger for the security 'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ], 'level': 'DEBUG', 'propagate': False }, 'business_events': { 'handlers': ['file_business_events', 'graylog'], 'level': 'DEBUG', 'propagate': False }, # Single tuning logger 'tuning': { 'handlers': ['tuning_file', 'graylog'] if env == 'production' else ['tuning_file'], 'level': 'DEBUG', 'propagate': False, }, '': { # root logger 'handlers': ['console'], 'level': 'WARNING', # Set higher level for root to minimize noise 'propagate': False }, } }