- Adaptations to support secure Redis Access
- Redis Connection Pooling set up for Celery, dogpile caching and flask session
This commit is contained in:
Josako
33
common/utils/cache/regions.py
vendored
33
common/utils/cache/regions.py
vendored
@@ -21,6 +21,18 @@ def get_redis_config(app):
|
||||
'redis_expiration_time': 3600,
|
||||
'distributed_lock': True,
|
||||
'thread_local_lock': False,
|
||||
# Ingebouwde connection pooling parameters
|
||||
'connection_pool_class': 'redis.BlockingConnectionPool',
|
||||
'connection_pool_class_kwargs': {
|
||||
'max_connections': 20,
|
||||
'timeout': 20,
|
||||
'retry_on_timeout': True,
|
||||
'socket_connect_timeout': 5,
|
||||
'socket_timeout': 5,
|
||||
},
|
||||
|
||||
# Key prefix voor namespace isolation
|
||||
'key_mangler': lambda key: f"cache:workers:{key}"
|
||||
}
|
||||
|
||||
# Add authentication if provided
|
||||
@@ -30,6 +42,27 @@ def get_redis_config(app):
|
||||
'password': redis_uri.password
|
||||
})
|
||||
|
||||
# SSL support using Dogpile's built-in mechanism
|
||||
cert_data = app.config.get('REDIS_CERT_DATA')
|
||||
if cert_data and redis_uri.scheme == 'rediss':
|
||||
import ssl
|
||||
import tempfile
|
||||
|
||||
# Create SSL context
|
||||
ssl_context = ssl.create_default_context()
|
||||
ssl_context.verify_mode = ssl.CERT_REQUIRED
|
||||
ssl_context.check_hostname = True
|
||||
|
||||
# Write cert to temp file
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as f:
|
||||
f.write(cert_data)
|
||||
ssl_cert_path = f.name
|
||||
|
||||
ssl_context.load_verify_locations(ssl_cert_path)
|
||||
|
||||
# Add SSL to connection pool kwargs
|
||||
config['connection_pool_class_kwargs']['ssl'] = ssl_context
|
||||
|
||||
return config
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import tempfile
|
||||
|
||||
from celery import Celery
|
||||
from kombu import Queue
|
||||
from werkzeug.local import LocalProxy
|
||||
@@ -6,6 +8,16 @@ from redbeat import RedBeatScheduler
|
||||
celery_app = Celery()
|
||||
|
||||
|
||||
def _create_ssl_cert_file(cert_data: str) -> str:
|
||||
"""Create temporary certificate file for Celery SSL"""
|
||||
if not cert_data:
|
||||
return None
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as cert_file:
|
||||
cert_file.write(cert_data)
|
||||
return cert_file.name
|
||||
|
||||
|
||||
def init_celery(celery, app, is_beat=False):
|
||||
celery_app.main = app.name
|
||||
|
||||
@@ -19,6 +31,32 @@ def init_celery(celery, app, is_beat=False):
|
||||
'enable_utc': app.config.get('CELERY_ENABLE_UTC', True),
|
||||
}
|
||||
|
||||
# Add broker transport options for SSL and connection pooling
|
||||
broker_transport_options = {
|
||||
'master_name': None,
|
||||
'max_connections': 20,
|
||||
'retry_on_timeout': True,
|
||||
'socket_connect_timeout': 5,
|
||||
'socket_timeout': 5,
|
||||
}
|
||||
|
||||
cert_data = app.config.get('REDIS_CERT_DATA')
|
||||
if cert_data:
|
||||
try:
|
||||
ssl_cert_file = _create_ssl_cert_file(cert_data)
|
||||
if ssl_cert_file:
|
||||
broker_transport_options.update({
|
||||
'ssl_cert_reqs': 'required',
|
||||
'ssl_ca_certs': ssl_cert_file,
|
||||
'ssl_check_hostname': True,
|
||||
})
|
||||
app.logger.info("SSL configured for Celery Redis connection")
|
||||
except Exception as e:
|
||||
app.logger.error(f"Failed to configure SSL for Celery: {e}")
|
||||
|
||||
celery_config['broker_transport_options'] = broker_transport_options
|
||||
celery_config['result_backend_transport_options'] = broker_transport_options
|
||||
|
||||
if is_beat:
|
||||
# Add configurations specific to Beat scheduler
|
||||
celery_config['beat_scheduler'] = 'redbeat.RedBeatScheduler'
|
||||
|
||||
195
config/config.py
195
config/config.py
@@ -14,7 +14,7 @@ class Config(object):
|
||||
SECRET_KEY = environ.get('SECRET_KEY')
|
||||
COMPONENT_NAME = environ.get('COMPONENT_NAME')
|
||||
|
||||
# Database Settings
|
||||
# Database Settings ---------------------------------------------------------------------------
|
||||
DB_HOST = environ.get('DB_HOST')
|
||||
DB_USER = environ.get('DB_USER')
|
||||
DB_PASS = environ.get('DB_PASS')
|
||||
@@ -23,11 +23,60 @@ class Config(object):
|
||||
SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}'
|
||||
SQLALCHEMY_BINDS = {'public': SQLALCHEMY_DATABASE_URI}
|
||||
|
||||
# Redis Settings ------------------------------------------------------------------------------
|
||||
REDIS_URL = environ.get('REDIS_URL')
|
||||
REDIS_PORT = environ.get('REDIS_PORT', '6379')
|
||||
REDIS_USER = environ.get('REDIS_USER')
|
||||
REDIS_PASS = environ.get('REDIS_PASS')
|
||||
REDIS_CERT_DATA = environ.get('REDIS_CERT')
|
||||
|
||||
if not REDIS_CERT_DATA: # We are in a simple dev/test environment
|
||||
REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
|
||||
else: # We are in a scaleway environment, providing name, user and certificate
|
||||
REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
|
||||
|
||||
REDIS_PREFIXES = {
|
||||
'celery_app': 'celery:app:',
|
||||
'celery_chat': 'celery:chat:',
|
||||
'session': 'session:',
|
||||
'cache_workers': 'cache:workers:',
|
||||
'pubsub_execution': 'pubsub:execution:',
|
||||
'startup_ops': 'startup:ops:',
|
||||
}
|
||||
|
||||
# Celery Redis settings
|
||||
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
|
||||
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
|
||||
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/0'
|
||||
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/0'
|
||||
|
||||
# SSE PubSub settings
|
||||
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/0'
|
||||
|
||||
# eveai_model cache Redis setting
|
||||
MODEL_CACHE_URL = f'{REDIS_BASE_URI}/0'
|
||||
|
||||
# Session Settings with Redis -----------------------------------------------------------------
|
||||
SESSION_TYPE = 'redis'
|
||||
SESSION_PERMANENT = True
|
||||
SESSION_USE_SIGNER = True
|
||||
PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
|
||||
SESSION_REFRESH_EACH_REQUEST = True
|
||||
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/0')
|
||||
SESSION_KEY_PREFIX = f'session_{COMPONENT_NAME}:'
|
||||
SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session'
|
||||
SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen
|
||||
SESSION_COOKIE_PATH = '/'
|
||||
SESSION_COOKIE_HTTPONLY = True
|
||||
SESSION_COOKIE_SECURE = False # True voor production met HTTPS
|
||||
SESSION_COOKIE_SAMESITE = 'Lax'
|
||||
REMEMBER_COOKIE_SAMESITE = 'strict'
|
||||
|
||||
WTF_CSRF_ENABLED = True
|
||||
WTF_CSRF_TIME_LIMIT = None
|
||||
WTF_CSRF_SSL_STRICT = False # Set to True if using HTTPS
|
||||
|
||||
# flask-security-too settings
|
||||
# flask-security-too settings -----------------------------------------------------------------
|
||||
# SECURITY_URL_PREFIX = '/admin'
|
||||
SECURITY_LOGIN_URL = '/admin/login'
|
||||
SECURITY_LOGOUT_URL = '/admin/logout'
|
||||
@@ -62,10 +111,10 @@ class Config(object):
|
||||
SECURITY_CSRF_HEADER = 'X-XSRF-TOKEN'
|
||||
WTF_CSRF_CHECK_DEFAULT = False
|
||||
|
||||
# file upload settings
|
||||
# file upload settings ------------------------------------------------------------------------
|
||||
MAX_CONTENT_LENGTH = 50 * 1024 * 1024
|
||||
|
||||
# supported languages
|
||||
# supported languages -------------------------------------------------------------------------
|
||||
SUPPORTED_LANGUAGE_DETAILS = {
|
||||
"English": {
|
||||
"iso 639-1": "en",
|
||||
@@ -152,10 +201,10 @@ class Config(object):
|
||||
SUPPORTED_LANGUAGES_FULL = list(SUPPORTED_LANGUAGE_DETAILS.keys())
|
||||
SUPPORTED_LANGUAGE_ISO639_1_LOOKUP = {lang_details["iso 639-1"]: lang_name for lang_name, lang_details in SUPPORTED_LANGUAGE_DETAILS.items()}
|
||||
|
||||
# supported currencies
|
||||
# supported currencies ------------------------------------------------------------------------
|
||||
SUPPORTED_CURRENCIES = ['€', '$']
|
||||
|
||||
# supported LLMs
|
||||
# supported LLMs & settings -------------------------------------------------------------------
|
||||
# SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
|
||||
SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed']
|
||||
SUPPORTED_LLMS = ['mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest']
|
||||
@@ -167,57 +216,21 @@ class Config(object):
|
||||
OPENAI_API_KEY = environ.get('OPENAI_API_KEY')
|
||||
MISTRAL_API_KEY = environ.get('MISTRAL_API_KEY')
|
||||
|
||||
# Celery settings
|
||||
# Celery settings (see above for Redis settings) ----------------------------------------------
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_RESULT_SERIALIZER = 'json'
|
||||
CELERY_ACCEPT_CONTENT = ['json']
|
||||
CELERY_TIMEZONE = 'UTC'
|
||||
CELERY_ENABLE_UTC = True
|
||||
|
||||
# SocketIO settings
|
||||
# SOCKETIO_ASYNC_MODE = 'threading'
|
||||
# SOCKETIO_ASYNC_MODE = 'gevent'
|
||||
|
||||
# Session Settings
|
||||
SESSION_TYPE = 'redis'
|
||||
SESSION_PERMANENT = True
|
||||
SESSION_USE_SIGNER = True
|
||||
PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
|
||||
SESSION_REFRESH_EACH_REQUEST = True
|
||||
|
||||
SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session'
|
||||
SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen
|
||||
SESSION_COOKIE_PATH = '/'
|
||||
SESSION_COOKIE_HTTPONLY = True
|
||||
SESSION_COOKIE_SECURE = False # True voor production met HTTPS
|
||||
SESSION_COOKIE_SAMESITE = 'Lax'
|
||||
REMEMBER_COOKIE_SAMESITE = 'strict'
|
||||
SESSION_KEY_PREFIX = f'{COMPONENT_NAME}_'
|
||||
|
||||
# JWT settings
|
||||
# JWT settings --------------------------------------------------------------------------------
|
||||
JWT_SECRET_KEY = environ.get('JWT_SECRET_KEY')
|
||||
JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1) # Set token expiry to 1 hour
|
||||
JWT_ACCESS_TOKEN_EXPIRES_DEPLOY = timedelta(hours=24) # Set long-lived token for deployment
|
||||
|
||||
# API Encryption
|
||||
# API Encryption ------------------------------------------------------------------------------
|
||||
API_ENCRYPTION_KEY = environ.get('API_ENCRYPTION_KEY')
|
||||
|
||||
# Fallback Algorithms
|
||||
FALLBACK_ALGORITHMS = [
|
||||
"RAG_TENANT",
|
||||
"RAG_WIKIPEDIA",
|
||||
"RAG_GOOGLE",
|
||||
"LLM"
|
||||
]
|
||||
|
||||
# Interaction algorithms
|
||||
INTERACTION_ALGORITHMS = {
|
||||
"RAG_TENANT": {"name": "RAG_TENANT", "description": "Algorithm using only information provided by the tenant"},
|
||||
"RAG_WIKIPEDIA": {"name": "RAG_WIKIPEDIA", "description": "Algorithm using information provided by Wikipedia"},
|
||||
"RAG_GOOGLE": {"name": "RAG_GOOGLE", "description": "Algorithm using information provided by Google"},
|
||||
"LLM": {"name": "LLM", "description": "Algorithm using information integrated in the used LLM"}
|
||||
}
|
||||
|
||||
# Email settings for API key notifications
|
||||
PROMOTIONAL_IMAGE_URL = 'https://askeveai.com/wp-content/uploads/2024/07/Evie-Call-scaled.jpg' # Replace with your actual URL
|
||||
|
||||
@@ -274,46 +287,6 @@ class DevConfig(Config):
|
||||
EVEAI_CHAT_LOCATION_PREFIX = '/chat'
|
||||
CHAT_CLIENT_PREFIX = 'chat-client/chat/'
|
||||
|
||||
# file upload settings
|
||||
# UPLOAD_FOLDER = '/app/tenant_files'
|
||||
|
||||
# Redis Settings
|
||||
REDIS_URL = 'redis'
|
||||
REDIS_PORT = '6379'
|
||||
REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
|
||||
REDIS_CERT_DATA = environ.get('REDIS_CERT')
|
||||
|
||||
# TODO: Redis certificaat inbouwen
|
||||
# Snippet:
|
||||
# import ssl
|
||||
# import redis
|
||||
#
|
||||
# # In je Redis connectie configuratie
|
||||
# if REDIS_CERT_DATA:
|
||||
# ssl_context = ssl.create_default_context()
|
||||
# ssl_context.check_hostname = False
|
||||
# ssl_context.verify_mode = ssl.CERT_NONE
|
||||
#
|
||||
# # Custom SSL context voor Redis
|
||||
# SESSION_REDIS = redis.from_url(REDIS_BASE_URI, ssl=ssl_context)
|
||||
|
||||
# Celery settings
|
||||
# eveai_app Redis Settings
|
||||
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
|
||||
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
|
||||
# eveai_chat Redis Settings
|
||||
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
|
||||
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
|
||||
# eveai_chat_workers cache Redis Settings
|
||||
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
|
||||
# specialist execution pub/sub Redis Settings
|
||||
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
|
||||
# eveai_model cache Redis setting
|
||||
MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'
|
||||
|
||||
# Session settings
|
||||
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
|
||||
|
||||
# PATH settings
|
||||
ffmpeg_path = '/usr/bin/ffmpeg'
|
||||
|
||||
@@ -338,33 +311,6 @@ class StagingConfig(Config):
|
||||
EVEAI_CHAT_LOCATION_PREFIX = '/chat'
|
||||
CHAT_CLIENT_PREFIX = 'chat-client/chat/'
|
||||
|
||||
# file upload settings
|
||||
# UPLOAD_FOLDER = '/app/tenant_files'
|
||||
|
||||
# Redis Settings
|
||||
REDIS_URL = environ.get('REDIS_URL')
|
||||
REDIS_PORT = environ.get('REDIS_PORT', '6379')
|
||||
REDIS_USER = environ.get('REDIS_USER')
|
||||
REDIS_PASS = environ.get('REDIS_PASS')
|
||||
REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
|
||||
|
||||
# Celery settings
|
||||
# eveai_app Redis Settings
|
||||
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
|
||||
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
|
||||
# eveai_chat Redis Settings
|
||||
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
|
||||
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
|
||||
# eveai_chat_workers cache Redis Settings
|
||||
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
|
||||
# specialist execution pub/sub Redis Settings
|
||||
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
|
||||
# eveai_model cache Redis setting
|
||||
MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'
|
||||
|
||||
# Session settings
|
||||
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
|
||||
|
||||
# PATH settings
|
||||
ffmpeg_path = '/usr/bin/ffmpeg'
|
||||
|
||||
@@ -398,31 +344,6 @@ class ProdConfig(Config):
|
||||
MAIL_USERNAME = 'eveai_super@flow-it.net'
|
||||
MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*'
|
||||
|
||||
# file upload settings
|
||||
# UPLOAD_FOLDER = '/app/tenant_files'
|
||||
|
||||
# Redis Settings
|
||||
REDIS_USER = environ.get('REDIS_USER')
|
||||
REDIS_PASS = environ.get('REDIS_PASS')
|
||||
REDIS_URL = environ.get('REDIS_URL')
|
||||
REDIS_PORT = environ.get('REDIS_PORT', '6379')
|
||||
REDIS_BASE_URI = f'redis://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
|
||||
|
||||
# Celery settings
|
||||
# eveai_app Redis Settings
|
||||
CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
|
||||
CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
|
||||
# eveai_chat Redis Settings
|
||||
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
|
||||
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
|
||||
# eveai_chat_workers cache Redis Settings
|
||||
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
|
||||
# specialist execution pub/sub Redis Settings
|
||||
SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
|
||||
|
||||
# Session settings
|
||||
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
|
||||
|
||||
# PATH settings
|
||||
ffmpeg_path = '/usr/bin/ffmpeg'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user