pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Adaptations to support secure Redis Access

Browse Source 
- Redis Connection Pooling set up for Celery, dogpile caching and flask session
This commit is contained in:
Josako
2025-08-31 17:43:30 +02:00
parent 25ab9ccf23
commit 35f58f0c57
3 changed files with 129 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
common/utils/cache/regions.py vendored
View File

@@ -21,6 +21,18 @@ def get_redis_config(app):
'redis_expiration_time': 3600, 'redis_expiration_time': 3600,
'distributed_lock': True, 'distributed_lock': True,
'thread_local_lock': False, 'thread_local_lock': False,
# Ingebouwde connection pooling parameters
'connection_pool_class': 'redis.BlockingConnectionPool',
'connection_pool_class_kwargs': {
'max_connections': 20,
'timeout': 20,
'retry_on_timeout': True,
'socket_connect_timeout': 5,
'socket_timeout': 5,
},
# Key prefix voor namespace isolation
'key_mangler': lambda key: f"cache:workers:{key}"
} }
# Add authentication if provided # Add authentication if provided
@@ -30,6 +42,27 @@ def get_redis_config(app):
'password': redis_uri.password 'password': redis_uri.password
}) })
# SSL support using Dogpile's built-in mechanism
cert_data = app.config.get('REDIS_CERT_DATA')
if cert_data and redis_uri.scheme == 'rediss':
import ssl
import tempfile
# Create SSL context
ssl_context = ssl.create_default_context()
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.check_hostname = True
# Write cert to temp file
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as f:
f.write(cert_data)
ssl_cert_path = f.name
ssl_context.load_verify_locations(ssl_cert_path)
# Add SSL to connection pool kwargs
config['connection_pool_class_kwargs']['ssl'] = ssl_context
return config return config

38
common/utils/celery_utils.py
View File

@@ -1,3 +1,5 @@
import tempfile
from celery import Celery from celery import Celery
from kombu import Queue from kombu import Queue
from werkzeug.local import LocalProxy from werkzeug.local import LocalProxy
@@ -6,6 +8,16 @@ from redbeat import RedBeatScheduler
celery_app = Celery() celery_app = Celery()
def _create_ssl_cert_file(cert_data: str) -> str:
"""Create temporary certificate file for Celery SSL"""
if not cert_data:
return None
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as cert_file:
cert_file.write(cert_data)
return cert_file.name
def init_celery(celery, app, is_beat=False): def init_celery(celery, app, is_beat=False):
celery_app.main = app.name celery_app.main = app.name
@@ -19,6 +31,32 @@ def init_celery(celery, app, is_beat=False):
'enable_utc': app.config.get('CELERY_ENABLE_UTC', True), 'enable_utc': app.config.get('CELERY_ENABLE_UTC', True),
} }
# Add broker transport options for SSL and connection pooling
broker_transport_options = {
'master_name': None,
'max_connections': 20,
'retry_on_timeout': True,
'socket_connect_timeout': 5,
'socket_timeout': 5,
}
cert_data = app.config.get('REDIS_CERT_DATA')
if cert_data:
try:
ssl_cert_file = _create_ssl_cert_file(cert_data)
if ssl_cert_file:
broker_transport_options.update({
'ssl_cert_reqs': 'required',
'ssl_ca_certs': ssl_cert_file,
'ssl_check_hostname': True,
})
app.logger.info("SSL configured for Celery Redis connection")
except Exception as e:
app.logger.error(f"Failed to configure SSL for Celery: {e}")
celery_config['broker_transport_options'] = broker_transport_options
celery_config['result_backend_transport_options'] = broker_transport_options
if is_beat: if is_beat:
# Add configurations specific to Beat scheduler # Add configurations specific to Beat scheduler
celery_config['beat_scheduler'] = 'redbeat.RedBeatScheduler' celery_config['beat_scheduler'] = 'redbeat.RedBeatScheduler'

195
config/config.py
View File

@@ -14,7 +14,7 @@ class Config(object):
SECRET_KEY = environ.get('SECRET_KEY') SECRET_KEY = environ.get('SECRET_KEY')
COMPONENT_NAME = environ.get('COMPONENT_NAME') COMPONENT_NAME = environ.get('COMPONENT_NAME')
# Database Settings # Database Settings ---------------------------------------------------------------------------
DB_HOST = environ.get('DB_HOST') DB_HOST = environ.get('DB_HOST')
DB_USER = environ.get('DB_USER') DB_USER = environ.get('DB_USER')
DB_PASS = environ.get('DB_PASS') DB_PASS = environ.get('DB_PASS')
@@ -23,11 +23,60 @@ class Config(object):
SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}' SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}'
SQLALCHEMY_BINDS = {'public': SQLALCHEMY_DATABASE_URI} SQLALCHEMY_BINDS = {'public': SQLALCHEMY_DATABASE_URI}
# Redis Settings ------------------------------------------------------------------------------
REDIS_URL = environ.get('REDIS_URL')
REDIS_PORT = environ.get('REDIS_PORT', '6379')
REDIS_USER = environ.get('REDIS_USER')
REDIS_PASS = environ.get('REDIS_PASS')
REDIS_CERT_DATA = environ.get('REDIS_CERT')
if not REDIS_CERT_DATA: # We are in a simple dev/test environment
REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
else: # We are in a scaleway environment, providing name, user and certificate
REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
REDIS_PREFIXES = {
'celery_app': 'celery:app:',
'celery_chat': 'celery:chat:',
'session': 'session:',
'cache_workers': 'cache:workers:',
'pubsub_execution': 'pubsub:execution:',
'startup_ops': 'startup:ops:',
}
# Celery Redis settings
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/0'
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/0'
# SSE PubSub settings
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/0'
# eveai_model cache Redis setting
MODEL_CACHE_URL = f'{REDIS_BASE_URI}/0'
# Session Settings with Redis -----------------------------------------------------------------
SESSION_TYPE = 'redis'
SESSION_PERMANENT = True
SESSION_USE_SIGNER = True
PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
SESSION_REFRESH_EACH_REQUEST = True
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/0')
SESSION_KEY_PREFIX = f'session_{COMPONENT_NAME}:'
SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session'
SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen
SESSION_COOKIE_PATH = '/'
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SECURE = False # True voor production met HTTPS
SESSION_COOKIE_SAMESITE = 'Lax'
REMEMBER_COOKIE_SAMESITE = 'strict'
WTF_CSRF_ENABLED = True WTF_CSRF_ENABLED = True
WTF_CSRF_TIME_LIMIT = None WTF_CSRF_TIME_LIMIT = None
WTF_CSRF_SSL_STRICT = False # Set to True if using HTTPS WTF_CSRF_SSL_STRICT = False # Set to True if using HTTPS
# flask-security-too settings # flask-security-too settings -----------------------------------------------------------------
# SECURITY_URL_PREFIX = '/admin' # SECURITY_URL_PREFIX = '/admin'
SECURITY_LOGIN_URL = '/admin/login' SECURITY_LOGIN_URL = '/admin/login'
SECURITY_LOGOUT_URL = '/admin/logout' SECURITY_LOGOUT_URL = '/admin/logout'
@@ -62,10 +111,10 @@ class Config(object):
SECURITY_CSRF_HEADER = 'X-XSRF-TOKEN' SECURITY_CSRF_HEADER = 'X-XSRF-TOKEN'
WTF_CSRF_CHECK_DEFAULT = False WTF_CSRF_CHECK_DEFAULT = False
# file upload settings # file upload settings ------------------------------------------------------------------------
MAX_CONTENT_LENGTH = 50 * 1024 * 1024 MAX_CONTENT_LENGTH = 50 * 1024 * 1024
# supported languages # supported languages -------------------------------------------------------------------------
SUPPORTED_LANGUAGE_DETAILS = { SUPPORTED_LANGUAGE_DETAILS = {
"English": { "English": {
"iso 639-1": "en", "iso 639-1": "en",
@@ -152,10 +201,10 @@ class Config(object):
SUPPORTED_LANGUAGES_FULL = list(SUPPORTED_LANGUAGE_DETAILS.keys()) SUPPORTED_LANGUAGES_FULL = list(SUPPORTED_LANGUAGE_DETAILS.keys())
SUPPORTED_LANGUAGE_ISO639_1_LOOKUP = {lang_details["iso 639-1"]: lang_name for lang_name, lang_details in SUPPORTED_LANGUAGE_DETAILS.items()} SUPPORTED_LANGUAGE_ISO639_1_LOOKUP = {lang_details["iso 639-1"]: lang_name for lang_name, lang_details in SUPPORTED_LANGUAGE_DETAILS.items()}
# supported currencies # supported currencies ------------------------------------------------------------------------
SUPPORTED_CURRENCIES = ['€', '$'] SUPPORTED_CURRENCIES = ['€', '$']
# supported LLMs # supported LLMs & settings -------------------------------------------------------------------
# SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed'] # SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed'] SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed']
SUPPORTED_LLMS = ['mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest'] SUPPORTED_LLMS = ['mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest']
@@ -167,57 +216,21 @@ class Config(object):
OPENAI_API_KEY = environ.get('OPENAI_API_KEY') OPENAI_API_KEY = environ.get('OPENAI_API_KEY')
MISTRAL_API_KEY = environ.get('MISTRAL_API_KEY') MISTRAL_API_KEY = environ.get('MISTRAL_API_KEY')
# Celery settings # Celery settings (see above for Redis settings) ----------------------------------------------
CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json'] CELERY_ACCEPT_CONTENT = ['json']
CELERY_TIMEZONE = 'UTC' CELERY_TIMEZONE = 'UTC'
CELERY_ENABLE_UTC = True CELERY_ENABLE_UTC = True
# SocketIO settings # JWT settings --------------------------------------------------------------------------------
# SOCKETIO_ASYNC_MODE = 'threading'
# SOCKETIO_ASYNC_MODE = 'gevent'
# Session Settings
SESSION_TYPE = 'redis'
SESSION_PERMANENT = True
SESSION_USE_SIGNER = True
PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
SESSION_REFRESH_EACH_REQUEST = True
SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session'
SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen
SESSION_COOKIE_PATH = '/'
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SECURE = False # True voor production met HTTPS
SESSION_COOKIE_SAMESITE = 'Lax'
REMEMBER_COOKIE_SAMESITE = 'strict'
SESSION_KEY_PREFIX = f'{COMPONENT_NAME}_'
# JWT settings
JWT_SECRET_KEY = environ.get('JWT_SECRET_KEY') JWT_SECRET_KEY = environ.get('JWT_SECRET_KEY')
JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1) # Set token expiry to 1 hour JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1) # Set token expiry to 1 hour
JWT_ACCESS_TOKEN_EXPIRES_DEPLOY = timedelta(hours=24) # Set long-lived token for deployment JWT_ACCESS_TOKEN_EXPIRES_DEPLOY = timedelta(hours=24) # Set long-lived token for deployment
# API Encryption # API Encryption ------------------------------------------------------------------------------
API_ENCRYPTION_KEY = environ.get('API_ENCRYPTION_KEY') API_ENCRYPTION_KEY = environ.get('API_ENCRYPTION_KEY')
# Fallback Algorithms
FALLBACK_ALGORITHMS = [
"RAG_TENANT",
"RAG_WIKIPEDIA",
"RAG_GOOGLE",
"LLM"
]
# Interaction algorithms
INTERACTION_ALGORITHMS = {
"RAG_TENANT": {"name": "RAG_TENANT", "description": "Algorithm using only information provided by the tenant"},
"RAG_WIKIPEDIA": {"name": "RAG_WIKIPEDIA", "description": "Algorithm using information provided by Wikipedia"},
"RAG_GOOGLE": {"name": "RAG_GOOGLE", "description": "Algorithm using information provided by Google"},
"LLM": {"name": "LLM", "description": "Algorithm using information integrated in the used LLM"}
}
# Email settings for API key notifications # Email settings for API key notifications
PROMOTIONAL_IMAGE_URL = 'https://askeveai.com/wp-content/uploads/2024/07/Evie-Call-scaled.jpg' # Replace with your actual URL PROMOTIONAL_IMAGE_URL = 'https://askeveai.com/wp-content/uploads/2024/07/Evie-Call-scaled.jpg' # Replace with your actual URL
@@ -274,46 +287,6 @@ class DevConfig(Config):
EVEAI_CHAT_LOCATION_PREFIX = '/chat' EVEAI_CHAT_LOCATION_PREFIX = '/chat'
CHAT_CLIENT_PREFIX = 'chat-client/chat/' CHAT_CLIENT_PREFIX = 'chat-client/chat/'
# file upload settings
# UPLOAD_FOLDER = '/app/tenant_files'
# Redis Settings
REDIS_URL = 'redis'
REDIS_PORT = '6379'
REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
REDIS_CERT_DATA = environ.get('REDIS_CERT')
# TODO: Redis certificaat inbouwen
# Snippet:
# import ssl
# import redis
#
# # In je Redis connectie configuratie
# if REDIS_CERT_DATA:
# ssl_context = ssl.create_default_context()
# ssl_context.check_hostname = False
# ssl_context.verify_mode = ssl.CERT_NONE
#
# # Custom SSL context voor Redis
# SESSION_REDIS = redis.from_url(REDIS_BASE_URI, ssl=ssl_context)
# Celery settings
# eveai_app Redis Settings
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
# eveai_chat Redis Settings
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
# eveai_chat_workers cache Redis Settings
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
# specialist execution pub/sub Redis Settings
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
# eveai_model cache Redis setting
MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'
# Session settings
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
# PATH settings # PATH settings
ffmpeg_path = '/usr/bin/ffmpeg' ffmpeg_path = '/usr/bin/ffmpeg'
@@ -338,33 +311,6 @@ class StagingConfig(Config):
EVEAI_CHAT_LOCATION_PREFIX = '/chat' EVEAI_CHAT_LOCATION_PREFIX = '/chat'
CHAT_CLIENT_PREFIX = 'chat-client/chat/' CHAT_CLIENT_PREFIX = 'chat-client/chat/'
# file upload settings
# UPLOAD_FOLDER = '/app/tenant_files'
# Redis Settings
REDIS_URL = environ.get('REDIS_URL')
REDIS_PORT = environ.get('REDIS_PORT', '6379')
REDIS_USER = environ.get('REDIS_USER')
REDIS_PASS = environ.get('REDIS_PASS')
REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
# Celery settings
# eveai_app Redis Settings
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
# eveai_chat Redis Settings
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
# eveai_chat_workers cache Redis Settings
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
# specialist execution pub/sub Redis Settings
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
# eveai_model cache Redis setting
MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'
# Session settings
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
# PATH settings # PATH settings
ffmpeg_path = '/usr/bin/ffmpeg' ffmpeg_path = '/usr/bin/ffmpeg'
@@ -398,31 +344,6 @@ class ProdConfig(Config):
MAIL_USERNAME = 'eveai_super@flow-it.net' MAIL_USERNAME = 'eveai_super@flow-it.net'
MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*' MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*'
# file upload settings
# UPLOAD_FOLDER = '/app/tenant_files'
# Redis Settings
REDIS_USER = environ.get('REDIS_USER')
REDIS_PASS = environ.get('REDIS_PASS')
REDIS_URL = environ.get('REDIS_URL')
REDIS_PORT = environ.get('REDIS_PORT', '6379')
REDIS_BASE_URI = f'redis://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
# Celery settings
# eveai_app Redis Settings
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
# eveai_chat Redis Settings
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
# eveai_chat_workers cache Redis Settings
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
# specialist execution pub/sub Redis Settings
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
# Session settings
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
# PATH settings # PATH settings
ffmpeg_path = '/usr/bin/ffmpeg' ffmpeg_path = '/usr/bin/ffmpeg'