diff --git a/common/utils/cache/regions.py b/common/utils/cache/regions.py index cfe48e0..52a05e1 100644 --- a/common/utils/cache/regions.py +++ b/common/utils/cache/regions.py @@ -21,6 +21,18 @@ def get_redis_config(app): 'redis_expiration_time': 3600, 'distributed_lock': True, 'thread_local_lock': False, + # Ingebouwde connection pooling parameters + 'connection_pool_class': 'redis.BlockingConnectionPool', + 'connection_pool_class_kwargs': { + 'max_connections': 20, + 'timeout': 20, + 'retry_on_timeout': True, + 'socket_connect_timeout': 5, + 'socket_timeout': 5, + }, + + # Key prefix voor namespace isolation + 'key_mangler': lambda key: f"cache:workers:{key}" } # Add authentication if provided @@ -30,6 +42,27 @@ def get_redis_config(app): 'password': redis_uri.password }) + # SSL support using Dogpile's built-in mechanism + cert_data = app.config.get('REDIS_CERT_DATA') + if cert_data and redis_uri.scheme == 'rediss': + import ssl + import tempfile + + # Create SSL context + ssl_context = ssl.create_default_context() + ssl_context.verify_mode = ssl.CERT_REQUIRED + ssl_context.check_hostname = True + + # Write cert to temp file + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as f: + f.write(cert_data) + ssl_cert_path = f.name + + ssl_context.load_verify_locations(ssl_cert_path) + + # Add SSL to connection pool kwargs + config['connection_pool_class_kwargs']['ssl'] = ssl_context + return config diff --git a/common/utils/celery_utils.py b/common/utils/celery_utils.py index 271cf39..d1bfa32 100644 --- a/common/utils/celery_utils.py +++ b/common/utils/celery_utils.py @@ -1,3 +1,5 @@ +import tempfile + from celery import Celery from kombu import Queue from werkzeug.local import LocalProxy @@ -6,6 +8,16 @@ from redbeat import RedBeatScheduler celery_app = Celery() +def _create_ssl_cert_file(cert_data: str) -> str: + """Create temporary certificate file for Celery SSL""" + if not cert_data: + return None + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as cert_file: + cert_file.write(cert_data) + return cert_file.name + + def init_celery(celery, app, is_beat=False): celery_app.main = app.name @@ -19,6 +31,32 @@ def init_celery(celery, app, is_beat=False): 'enable_utc': app.config.get('CELERY_ENABLE_UTC', True), } + # Add broker transport options for SSL and connection pooling + broker_transport_options = { + 'master_name': None, + 'max_connections': 20, + 'retry_on_timeout': True, + 'socket_connect_timeout': 5, + 'socket_timeout': 5, + } + + cert_data = app.config.get('REDIS_CERT_DATA') + if cert_data: + try: + ssl_cert_file = _create_ssl_cert_file(cert_data) + if ssl_cert_file: + broker_transport_options.update({ + 'ssl_cert_reqs': 'required', + 'ssl_ca_certs': ssl_cert_file, + 'ssl_check_hostname': True, + }) + app.logger.info("SSL configured for Celery Redis connection") + except Exception as e: + app.logger.error(f"Failed to configure SSL for Celery: {e}") + + celery_config['broker_transport_options'] = broker_transport_options + celery_config['result_backend_transport_options'] = broker_transport_options + if is_beat: # Add configurations specific to Beat scheduler celery_config['beat_scheduler'] = 'redbeat.RedBeatScheduler' diff --git a/config/config.py b/config/config.py index e8c0ced..6d022d9 100644 --- a/config/config.py +++ b/config/config.py @@ -14,7 +14,7 @@ class Config(object): SECRET_KEY = environ.get('SECRET_KEY') COMPONENT_NAME = environ.get('COMPONENT_NAME') - # Database Settings + # Database Settings --------------------------------------------------------------------------- DB_HOST = environ.get('DB_HOST') DB_USER = environ.get('DB_USER') DB_PASS = environ.get('DB_PASS') @@ -23,11 +23,60 @@ class Config(object): SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}' SQLALCHEMY_BINDS = {'public': SQLALCHEMY_DATABASE_URI} + # Redis Settings ------------------------------------------------------------------------------ + REDIS_URL = environ.get('REDIS_URL') + REDIS_PORT = environ.get('REDIS_PORT', '6379') + REDIS_USER = environ.get('REDIS_USER') + REDIS_PASS = environ.get('REDIS_PASS') + REDIS_CERT_DATA = environ.get('REDIS_CERT') + + if not REDIS_CERT_DATA: # We are in a simple dev/test environment + REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}' + else: # We are in a scaleway environment, providing name, user and certificate + REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}' + + REDIS_PREFIXES = { + 'celery_app': 'celery:app:', + 'celery_chat': 'celery:chat:', + 'session': 'session:', + 'cache_workers': 'cache:workers:', + 'pubsub_execution': 'pubsub:execution:', + 'startup_ops': 'startup:ops:', + } + + # Celery Redis settings + CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0' + CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0' + CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/0' + CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/0' + + # SSE PubSub settings + SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/0' + + # eveai_model cache Redis setting + MODEL_CACHE_URL = f'{REDIS_BASE_URI}/0' + + # Session Settings with Redis ----------------------------------------------------------------- + SESSION_TYPE = 'redis' + SESSION_PERMANENT = True + SESSION_USE_SIGNER = True + PERMANENT_SESSION_LIFETIME = timedelta(minutes=60) + SESSION_REFRESH_EACH_REQUEST = True + SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/0') + SESSION_KEY_PREFIX = f'session_{COMPONENT_NAME}:' + SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session' + SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen + SESSION_COOKIE_PATH = '/' + SESSION_COOKIE_HTTPONLY = True + SESSION_COOKIE_SECURE = False # True voor production met HTTPS + SESSION_COOKIE_SAMESITE = 'Lax' + REMEMBER_COOKIE_SAMESITE = 'strict' + WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None WTF_CSRF_SSL_STRICT = False # Set to True if using HTTPS - # flask-security-too settings + # flask-security-too settings ----------------------------------------------------------------- # SECURITY_URL_PREFIX = '/admin' SECURITY_LOGIN_URL = '/admin/login' SECURITY_LOGOUT_URL = '/admin/logout' @@ -62,10 +111,10 @@ class Config(object): SECURITY_CSRF_HEADER = 'X-XSRF-TOKEN' WTF_CSRF_CHECK_DEFAULT = False - # file upload settings + # file upload settings ------------------------------------------------------------------------ MAX_CONTENT_LENGTH = 50 * 1024 * 1024 - # supported languages + # supported languages ------------------------------------------------------------------------- SUPPORTED_LANGUAGE_DETAILS = { "English": { "iso 639-1": "en", @@ -152,10 +201,10 @@ class Config(object): SUPPORTED_LANGUAGES_FULL = list(SUPPORTED_LANGUAGE_DETAILS.keys()) SUPPORTED_LANGUAGE_ISO639_1_LOOKUP = {lang_details["iso 639-1"]: lang_name for lang_name, lang_details in SUPPORTED_LANGUAGE_DETAILS.items()} - # supported currencies + # supported currencies ------------------------------------------------------------------------ SUPPORTED_CURRENCIES = ['€', '$'] - # supported LLMs + # supported LLMs & settings ------------------------------------------------------------------- # SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed'] SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed'] SUPPORTED_LLMS = ['mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest'] @@ -167,57 +216,21 @@ class Config(object): OPENAI_API_KEY = environ.get('OPENAI_API_KEY') MISTRAL_API_KEY = environ.get('MISTRAL_API_KEY') - # Celery settings + # Celery settings (see above for Redis settings) ---------------------------------------------- CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] CELERY_TIMEZONE = 'UTC' CELERY_ENABLE_UTC = True - # SocketIO settings - # SOCKETIO_ASYNC_MODE = 'threading' - # SOCKETIO_ASYNC_MODE = 'gevent' - - # Session Settings - SESSION_TYPE = 'redis' - SESSION_PERMANENT = True - SESSION_USE_SIGNER = True - PERMANENT_SESSION_LIFETIME = timedelta(minutes=60) - SESSION_REFRESH_EACH_REQUEST = True - - SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session' - SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen - SESSION_COOKIE_PATH = '/' - SESSION_COOKIE_HTTPONLY = True - SESSION_COOKIE_SECURE = False # True voor production met HTTPS - SESSION_COOKIE_SAMESITE = 'Lax' - REMEMBER_COOKIE_SAMESITE = 'strict' - SESSION_KEY_PREFIX = f'{COMPONENT_NAME}_' - - # JWT settings + # JWT settings -------------------------------------------------------------------------------- JWT_SECRET_KEY = environ.get('JWT_SECRET_KEY') JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1) # Set token expiry to 1 hour JWT_ACCESS_TOKEN_EXPIRES_DEPLOY = timedelta(hours=24) # Set long-lived token for deployment - # API Encryption + # API Encryption ------------------------------------------------------------------------------ API_ENCRYPTION_KEY = environ.get('API_ENCRYPTION_KEY') - # Fallback Algorithms - FALLBACK_ALGORITHMS = [ - "RAG_TENANT", - "RAG_WIKIPEDIA", - "RAG_GOOGLE", - "LLM" - ] - - # Interaction algorithms - INTERACTION_ALGORITHMS = { - "RAG_TENANT": {"name": "RAG_TENANT", "description": "Algorithm using only information provided by the tenant"}, - "RAG_WIKIPEDIA": {"name": "RAG_WIKIPEDIA", "description": "Algorithm using information provided by Wikipedia"}, - "RAG_GOOGLE": {"name": "RAG_GOOGLE", "description": "Algorithm using information provided by Google"}, - "LLM": {"name": "LLM", "description": "Algorithm using information integrated in the used LLM"} - } - # Email settings for API key notifications PROMOTIONAL_IMAGE_URL = 'https://askeveai.com/wp-content/uploads/2024/07/Evie-Call-scaled.jpg' # Replace with your actual URL @@ -274,46 +287,6 @@ class DevConfig(Config): EVEAI_CHAT_LOCATION_PREFIX = '/chat' CHAT_CLIENT_PREFIX = 'chat-client/chat/' - # file upload settings - # UPLOAD_FOLDER = '/app/tenant_files' - - # Redis Settings - REDIS_URL = 'redis' - REDIS_PORT = '6379' - REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}' - REDIS_CERT_DATA = environ.get('REDIS_CERT') - - # TODO: Redis certificaat inbouwen - # Snippet: - # import ssl - # import redis - # - # # In je Redis connectie configuratie - # if REDIS_CERT_DATA: - # ssl_context = ssl.create_default_context() - # ssl_context.check_hostname = False - # ssl_context.verify_mode = ssl.CERT_NONE - # - # # Custom SSL context voor Redis - # SESSION_REDIS = redis.from_url(REDIS_BASE_URI, ssl=ssl_context) - - # Celery settings - # eveai_app Redis Settings - CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0' - CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0' - # eveai_chat Redis Settings - CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' - CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' - # eveai_chat_workers cache Redis Settings - CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4' - # specialist execution pub/sub Redis Settings - SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5' - # eveai_model cache Redis setting - MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6' - - # Session settings - SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') - # PATH settings ffmpeg_path = '/usr/bin/ffmpeg' @@ -338,33 +311,6 @@ class StagingConfig(Config): EVEAI_CHAT_LOCATION_PREFIX = '/chat' CHAT_CLIENT_PREFIX = 'chat-client/chat/' - # file upload settings - # UPLOAD_FOLDER = '/app/tenant_files' - - # Redis Settings - REDIS_URL = environ.get('REDIS_URL') - REDIS_PORT = environ.get('REDIS_PORT', '6379') - REDIS_USER = environ.get('REDIS_USER') - REDIS_PASS = environ.get('REDIS_PASS') - REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}' - - # Celery settings - # eveai_app Redis Settings - CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0' - CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0' - # eveai_chat Redis Settings - CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' - CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' - # eveai_chat_workers cache Redis Settings - CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4' - # specialist execution pub/sub Redis Settings - SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5' - # eveai_model cache Redis setting - MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6' - - # Session settings - SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') - # PATH settings ffmpeg_path = '/usr/bin/ffmpeg' @@ -398,31 +344,6 @@ class ProdConfig(Config): MAIL_USERNAME = 'eveai_super@flow-it.net' MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*' - # file upload settings - # UPLOAD_FOLDER = '/app/tenant_files' - - # Redis Settings - REDIS_USER = environ.get('REDIS_USER') - REDIS_PASS = environ.get('REDIS_PASS') - REDIS_URL = environ.get('REDIS_URL') - REDIS_PORT = environ.get('REDIS_PORT', '6379') - REDIS_BASE_URI = f'redis://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}' - - # Celery settings - # eveai_app Redis Settings - CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0' - CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0' - # eveai_chat Redis Settings - CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' - CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' - # eveai_chat_workers cache Redis Settings - CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4' - # specialist execution pub/sub Redis Settings - SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5' - - # Session settings - SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') - # PATH settings ffmpeg_path = '/usr/bin/ffmpeg'