pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Improvements working with the cloud, minio, graylog and some first bugfixing

Browse Source
This commit is contained in:
Josako
2024-08-13 09:04:19 +02:00
parent 9caa423bcf
commit ab38dd7540
23 changed files with 414 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
config/config.py
View File

@@ -125,7 +125,6 @@ class DevConfig(Config):
DEVELOPMENT = True
DEBUG = True
FLASK_DEBUG = True
PYCHARM_DEBUG = False
EXPLAIN_TEMPLATE_LOADING = False
# Database Settings
@@ -191,6 +190,7 @@ class DevConfig(Config):
MINIO_ENDPOINT = 'minio:9000'
MINIO_ACCESS_KEY = 'minioadmin'
MINIO_SECRET_KEY = 'minioadmin'
MINIO_USE_HTTPS = False
class ProdConfig(Config):
@@ -198,7 +198,6 @@ class ProdConfig(Config):
DEBUG = False
DEBUG = False
FLASK_DEBUG = False
PYCHARM_DEBUG = False
EXPLAIN_TEMPLATE_LOADING = False
# Database Settings
@@ -267,6 +266,12 @@ class ProdConfig(Config):
# PATH settings
ffmpeg_path = '/usr/bin/ffmpeg'
# MINIO
MINIO_ENDPOINT = environ.get('MINIO_ENDPOINT')
MINIO_ACCESS_KEY = environ.get('MINIO_ACCESS_KEY')
MINIO_SECRET_KEY = environ.get('MINIO_SECRET_KEY')
MINIO_USE_HTTPS = True
def get_config(config_name='dev'):
configs = {

80
config/logging_config.py
View File

@@ -1,3 +1,29 @@
import os
from graypy import GELFUDPHandler
import logging
import logging.config
# Graylog configuration
GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost')
GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201))
env = os.environ.get('FLASK_ENV', 'development')
class CustomLogRecord(logging.LogRecord):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.component = os.environ.get('COMPONENT_NAME', 'eveai_app') # Set default component value here
def custom_log_record_factory(*args, **kwargs):
record = CustomLogRecord(*args, **kwargs)
return record
# Set the custom log record factory
logging.setLogRecordFactory(custom_log_record_factory)
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
@@ -6,7 +32,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_app.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -14,7 +40,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_workers.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -22,7 +48,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_chat.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -30,7 +56,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_chat_workers.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -38,7 +64,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/sqlalchemy.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -46,7 +72,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/mailman.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -54,7 +80,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/security.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -62,7 +88,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/rag_tuning.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -70,7 +96,7 @@ LOGGING = {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/embed_tuning.log',
'maxBytes': 1024*1024*5, # 5MB
'maxBytes': 1024 * 1024 * 5, # 5MB
'backupCount': 10,
'formatter': 'standard',
},
@@ -79,55 +105,69 @@ LOGGING = {
'level': 'DEBUG',
'formatter': 'standard',
},
'graylog': {
'level': 'DEBUG',
'class': 'graypy.GELFUDPHandler',
'host': GRAYLOG_HOST,
'port': GRAYLOG_PORT,
'debugging_fields': True, # Set to True if you want to include debugging fields
'extra_fields': True, # Set to True if you want to include extra fields
},
},
'formatters': {
'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
'[Thread: %(threadName)s] [Host: %(hostname)s]: %(message)s'
},
'graylog': {
'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
'[Thread: %(threadName)s] [Host: %(hostname)s]: %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S',
},
},
'loggers': {
'eveai_app': { # logger for the eveai_app
'handlers': ['file_app',],
'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ],
'level': 'DEBUG',
'propagate': False
},
'eveai_workers': { # logger for the eveai_workers
'handlers': ['file_workers',],
'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ],
'level': 'DEBUG',
'propagate': False
},
'eveai_chat': { # logger for the eveai_chat
'handlers': ['file_chat',],
'handlers': ['file_chat', 'graylog', ] if env == 'production' else ['file_chat', ],
'level': 'DEBUG',
'propagate': False
},
'eveai_chat_workers': { # logger for the eveai_chat_workers
'handlers': ['file_chat_workers',],
'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ],
'level': 'DEBUG',
'propagate': False
},
'sqlalchemy.engine': { # logger for the sqlalchemy
'handlers': ['file_sqlalchemy',],
'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ],
'level': 'DEBUG',
'propagate': False
},
'mailman': { # logger for the mailman
'handlers': ['file_mailman', 'console'],
'handlers': ['file_mailman', 'graylog', ] if env == 'production' else ['file_mailman', ],
'level': 'DEBUG',
'propagate': False
},
'security': { # logger for the security
'handlers': ['file_security', 'console'],
'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ],
'level': 'DEBUG',
'propagate': False
},
'rag_tuning': { # logger for the rag_tuning
'handlers': ['file_rag_tuning', 'console'],
'handlers': ['file_rag_tuning', 'graylog', ] if env == 'production' else ['file_rag_tuning', ],
'level': 'DEBUG',
'propagate': False
},
'embed_tuning': { # logger for the embed_tuning
'handlers': ['file_embed_tuning', 'console'],
'handlers': ['file_embed_tuning', 'graylog', ] if env == 'production' else ['file_embed_tuning', ],
'level': 'DEBUG',
'propagate': False
},
@@ -137,4 +177,4 @@ LOGGING = {
'propagate': False
},
}
}
}