488 lines
18 KiB
Python
488 lines
18 KiB
Python
import json
|
|
import os
|
|
from datetime import datetime as dt, timezone as tz
|
|
|
|
from flask import current_app
|
|
from graypy import GELFUDPHandler
|
|
import logging
|
|
import logging.config
|
|
|
|
# Graylog configuration
|
|
GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost')
|
|
GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201))
|
|
env = os.environ.get('FLASK_ENV', 'development')
|
|
|
|
|
|
def pad_string(s, target_length=100, pad_char='-'):
|
|
"""
|
|
Pads a string with the specified character until it reaches the target length.
|
|
|
|
Args:
|
|
s: The original string
|
|
target_length: The desired total length
|
|
pad_char: Character to use for padding
|
|
|
|
Returns:
|
|
The padded string
|
|
"""
|
|
current_length = len(s)
|
|
if current_length >= target_length:
|
|
return s
|
|
|
|
padding_needed = target_length - current_length - 1
|
|
return s + " " + (pad_char * padding_needed)
|
|
|
|
|
|
class TuningLogRecord(logging.LogRecord):
|
|
"""Extended LogRecord that handles both tuning and business event logging"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
# Initialize extra fields after parent initialization
|
|
self._extra_fields = {}
|
|
self._is_tuning_log = False
|
|
self._tuning_type = None
|
|
self._tuning_tenant_id = None
|
|
self._tuning_catalog_id = None
|
|
self._tuning_specialist_id = None
|
|
self._tuning_retriever_id = None
|
|
self._tuning_processor_id = None
|
|
self._session_id = None
|
|
self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')
|
|
|
|
def getMessage(self):
|
|
"""
|
|
Override getMessage to handle both string and dict messages
|
|
"""
|
|
msg = self.msg
|
|
if self.args:
|
|
msg = msg % self.args
|
|
return msg
|
|
|
|
@property
|
|
def is_tuning_log(self):
|
|
return self._is_tuning_log
|
|
|
|
@is_tuning_log.setter
|
|
def is_tuning_log(self, value):
|
|
object.__setattr__(self, '_is_tuning_log', value)
|
|
|
|
@property
|
|
def tuning_type(self):
|
|
return self._tuning_type
|
|
|
|
@tuning_type.setter
|
|
def tuning_type(self, value):
|
|
object.__setattr__(self, '_tuning_type', value)
|
|
|
|
def get_tuning_data(self):
|
|
"""Get all tuning-related data if this is a tuning log"""
|
|
if not self._is_tuning_log:
|
|
return {}
|
|
|
|
return {
|
|
'is_tuning_log': self._is_tuning_log,
|
|
'tuning_type': self._tuning_type,
|
|
'tuning_tenant_id': self._tuning_tenant_id,
|
|
'tuning_catalog_id': self._tuning_catalog_id,
|
|
'tuning_specialist_id': self._tuning_specialist_id,
|
|
'tuning_retriever_id': self._tuning_retriever_id,
|
|
'tuning_processor_id': self._tuning_processor_id,
|
|
'session_id': self._session_id,
|
|
}
|
|
|
|
def set_tuning_data(self, tenant_id=None, catalog_id=None, specialist_id=None,
|
|
retriever_id=None, processor_id=None, session_id=None,):
|
|
"""Set tuning-specific data"""
|
|
object.__setattr__(self, '_tuning_tenant_id', tenant_id)
|
|
object.__setattr__(self, '_tuning_catalog_id', catalog_id)
|
|
object.__setattr__(self, '_tuning_specialist_id', specialist_id)
|
|
object.__setattr__(self, '_tuning_retriever_id', retriever_id)
|
|
object.__setattr__(self, '_tuning_processor_id', processor_id)
|
|
object.__setattr__(self, '_session_id', session_id)
|
|
|
|
|
|
class TuningFormatter(logging.Formatter):
|
|
"""Universal formatter for all tuning logs"""
|
|
|
|
def __init__(self, fmt=None, datefmt=None):
|
|
super().__init__(fmt or '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
|
|
datefmt or '%Y-%m-%d %H:%M:%S')
|
|
|
|
def format(self, record):
|
|
# First format with the default formatter to handle basic fields
|
|
formatted_msg = super().format(record)
|
|
|
|
# If this is a tuning log, add the additional context
|
|
if getattr(record, 'is_tuning_log', False):
|
|
try:
|
|
identifiers = []
|
|
if hasattr(record, 'tenant_id') and record.tenant_id:
|
|
identifiers.append(f"Tenant: {record.tenant_id}")
|
|
if hasattr(record, 'catalog_id') and record.catalog_id:
|
|
identifiers.append(f"Catalog: {record.catalog_id}")
|
|
if hasattr(record, 'processor_id') and record.processor_id:
|
|
identifiers.append(f"Processor: {record.processor_id}")
|
|
if hasattr(record, 'specialist_id') and record.specialist_id:
|
|
identifiers.append(f"Specialist: {record.specialist_id}")
|
|
if hasattr(record, 'retriever_id') and record.retriever_id:
|
|
identifiers.append(f"Retriever: {record.retriever_id}")
|
|
if hasattr(record, 'session_id') and record.session_id:
|
|
identifiers.append(f"Session: {record.session_id}")
|
|
|
|
formatted_msg = (
|
|
f"{formatted_msg}\n"
|
|
f"[TUNING {record.tuning_type}] [{' | '.join(identifiers)}]"
|
|
)
|
|
|
|
if hasattr(record, 'tuning_data') and record.tuning_data:
|
|
formatted_msg += f"\nData: {json.dumps(record.tuning_data, indent=2)}"
|
|
|
|
except Exception as e:
|
|
return f"{formatted_msg} (Error formatting tuning data: {str(e)})"
|
|
|
|
return formatted_msg
|
|
|
|
|
|
class GraylogFormatter(logging.Formatter):
|
|
"""Maintains existing Graylog formatting while adding tuning fields"""
|
|
|
|
def format(self, record):
|
|
if getattr(record, 'is_tuning_log', False):
|
|
# Add tuning-specific fields to Graylog
|
|
record.tuning_fields = {
|
|
'is_tuning_log': True,
|
|
'tuning_type': record.tuning_type,
|
|
'tenant_id': record.tenant_id,
|
|
'catalog_id': record.catalog_id,
|
|
'specialist_id': record.specialist_id,
|
|
'retriever_id': record.retriever_id,
|
|
'processor_id': record.processor_id,
|
|
'session_id': record.session_id,
|
|
}
|
|
return super().format(record)
|
|
|
|
class TuningLogger:
|
|
"""Helper class to manage tuning logs with consistent structure"""
|
|
|
|
def __init__(self, logger_name, tenant_id=None, catalog_id=None, specialist_id=None, retriever_id=None,
|
|
processor_id=None, session_id=None, log_file=None):
|
|
"""
|
|
Initialize a tuning logger
|
|
|
|
Args:
|
|
logger_name: Base name for the logger
|
|
tenant_id: Optional tenant ID for context
|
|
catalog_id: Optional catalog ID for context
|
|
specialist_id: Optional specialist ID for context
|
|
retriever_id: Optional retriever ID for context
|
|
processor_id: Optional processor ID for context
|
|
session_id: Optional session ID for context and log file naming
|
|
log_file: Optional custom log file name to use
|
|
"""
|
|
|
|
self.logger = logging.getLogger(logger_name)
|
|
self.tenant_id = tenant_id
|
|
self.catalog_id = catalog_id
|
|
self.specialist_id = specialist_id
|
|
self.retriever_id = retriever_id
|
|
self.processor_id = processor_id
|
|
self.session_id = session_id
|
|
self.log_file = log_file
|
|
# Determine whether to use a session-specific logger
|
|
if session_id:
|
|
# Create a unique logger name for this session
|
|
session_logger_name = f"{logger_name}_{session_id}"
|
|
self.logger = logging.getLogger(session_logger_name)
|
|
|
|
# If this logger doesn't have handlers yet, configure it
|
|
if not self.logger.handlers:
|
|
# Determine log file path
|
|
if not log_file and session_id:
|
|
log_file = f"logs/tuning_{session_id}.log"
|
|
elif not log_file:
|
|
log_file = "logs/tuning.log"
|
|
|
|
# Configure the logger
|
|
self._configure_session_logger(log_file)
|
|
else:
|
|
# Use the standard tuning logger
|
|
self.logger = logging.getLogger(logger_name)
|
|
|
|
def _configure_session_logger(self, log_file):
|
|
"""Configure a new session-specific logger with appropriate handlers"""
|
|
# Create and configure a file handler
|
|
file_handler = logging.handlers.RotatingFileHandler(
|
|
filename=log_file,
|
|
maxBytes=1024 * 1024 * 3, # 3MB
|
|
backupCount=3
|
|
)
|
|
file_handler.setFormatter(TuningFormatter())
|
|
file_handler.setLevel(logging.DEBUG)
|
|
|
|
# Add the file handler to the logger
|
|
self.logger.addHandler(file_handler)
|
|
|
|
# Add Graylog handler in production
|
|
env = os.environ.get('FLASK_ENV', 'development')
|
|
if env == 'production':
|
|
try:
|
|
graylog_handler = GELFUDPHandler(
|
|
host=GRAYLOG_HOST,
|
|
port=GRAYLOG_PORT,
|
|
debugging_fields=True
|
|
)
|
|
graylog_handler.setFormatter(GraylogFormatter())
|
|
self.logger.addHandler(graylog_handler)
|
|
except Exception as e:
|
|
# Fall back to just file logging if Graylog setup fails
|
|
fallback_logger = logging.getLogger('eveai_app')
|
|
fallback_logger.warning(f"Failed to set up Graylog handler: {str(e)}")
|
|
|
|
# Set logger level and disable propagation
|
|
self.logger.setLevel(logging.DEBUG)
|
|
self.logger.propagate = False
|
|
|
|
|
|
def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DEBUG):
|
|
"""Log a tuning event with structured data"""
|
|
try:
|
|
# Create a standard LogRecord for tuning
|
|
record = logging.LogRecord(
|
|
name=self.logger.name,
|
|
level=level,
|
|
pathname='',
|
|
lineno=0,
|
|
msg=pad_string(message, 100, '-'),
|
|
args=(),
|
|
exc_info=None
|
|
)
|
|
|
|
# Add tuning-specific attributes
|
|
record.is_tuning_log = True
|
|
record.tuning_type = tuning_type
|
|
record.tenant_id = self.tenant_id
|
|
record.catalog_id = self.catalog_id
|
|
record.specialist_id = self.specialist_id
|
|
record.retriever_id = self.retriever_id
|
|
record.processor_id = self.processor_id
|
|
record.session_id = self.session_id
|
|
|
|
if data:
|
|
record.tuning_data = data
|
|
|
|
# Process the record
|
|
self.logger.handle(record)
|
|
|
|
except Exception as e:
|
|
fallback_logger = logging.getLogger('eveai_workers')
|
|
fallback_logger.exception(f"Failed to log tuning message: {str(e)}")
|
|
|
|
|
|
# Set the custom log record factory
|
|
logging.setLogRecordFactory(TuningLogRecord)
|
|
|
|
|
|
LOGGING = {
|
|
'version': 1,
|
|
'disable_existing_loggers': False,
|
|
'handlers': {
|
|
'file_app': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_app.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_workers': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_workers.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_chat_client': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_chat_client.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_chat_workers': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_chat_workers.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_api': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_api.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_beat': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_beat.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_entitlements': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/eveai_entitlements.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_sqlalchemy': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/sqlalchemy.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_security': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/security.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_rag_tuning': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/rag_tuning.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_embed_tuning': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/embed_tuning.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'file_business_events': {
|
|
'level': 'INFO',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/business_events.log',
|
|
'maxBytes': 1024 * 1024 * 1, # 1MB
|
|
'backupCount': 2,
|
|
'formatter': 'standard',
|
|
},
|
|
'console': {
|
|
'class': 'logging.StreamHandler',
|
|
'level': 'DEBUG',
|
|
'formatter': 'standard',
|
|
},
|
|
'tuning_file': {
|
|
'level': 'DEBUG',
|
|
'class': 'logging.handlers.RotatingFileHandler',
|
|
'filename': 'logs/tuning.log',
|
|
'maxBytes': 1024 * 1024 * 3, # 3MB
|
|
'backupCount': 3,
|
|
'formatter': 'tuning',
|
|
},
|
|
'graylog': {
|
|
'level': 'DEBUG',
|
|
'class': 'graypy.GELFUDPHandler',
|
|
'host': GRAYLOG_HOST,
|
|
'port': GRAYLOG_PORT,
|
|
'debugging_fields': True,
|
|
'formatter': 'graylog'
|
|
},
|
|
},
|
|
'formatters': {
|
|
'standard': {
|
|
'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d]: %(message)s',
|
|
'datefmt': '%Y-%m-%d %H:%M:%S'
|
|
},
|
|
'graylog': {
|
|
'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
|
|
'[Thread: %(threadName)s]: %(message)s',
|
|
'datefmt': '%Y-%m-%d %H:%M:%S',
|
|
'()': GraylogFormatter
|
|
},
|
|
'tuning': {
|
|
'()': TuningFormatter,
|
|
'datefmt': '%Y-%m-%d %H:%M:%S UTC'
|
|
}
|
|
},
|
|
'loggers': {
|
|
'eveai_app': { # logger for the eveai_app
|
|
'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'eveai_workers': { # logger for the eveai_workers
|
|
'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'eveai_chat_client': { # logger for the eveai_chat
|
|
'handlers': ['file_chat_client', 'graylog', ] if env == 'production' else ['file_chat_client', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'eveai_chat_workers': { # logger for the eveai_chat_workers
|
|
'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'eveai_api': { # logger for the eveai_chat_workers
|
|
'handlers': ['file_api', 'graylog', ] if env == 'production' else ['file_api', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'eveai_beat': { # logger for the eveai_beat
|
|
'handlers': ['file_beat', 'graylog', ] if env == 'production' else ['file_beat', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'eveai_entitlements': { # logger for the eveai_entitlements
|
|
'handlers': ['file_entitlements', 'graylog', ] if env == 'production' else ['file_entitlements', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'sqlalchemy.engine': { # logger for the sqlalchemy
|
|
'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'security': { # logger for the security
|
|
'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
'business_events': {
|
|
'handlers': ['file_business_events', 'graylog'],
|
|
'level': 'DEBUG',
|
|
'propagate': False
|
|
},
|
|
# Single tuning logger
|
|
'tuning': {
|
|
'handlers': ['tuning_file', 'graylog'] if env == 'production' else ['tuning_file'],
|
|
'level': 'DEBUG',
|
|
'propagate': False,
|
|
},
|
|
'': { # root logger
|
|
'handlers': ['console'],
|
|
'level': 'WARNING', # Set higher level for root to minimize noise
|
|
'propagate': False
|
|
},
|
|
}
|
|
}
|