From 54a96414406ee66704a5077803f206360cb45b6a Mon Sep 17 00:00:00 2001 From: Josako Date: Thu, 4 Sep 2025 15:22:45 +0200 Subject: [PATCH] - TLS Refactoring --- common/utils/cache/regions.py | 19 ++----- common/utils/celery_utils.py | 52 +++++-------------- config/config.py | 45 +++++++++++++++- .../Production Setup/cluster-install.md | 23 ++++---- documentation/generating_codes.md | 8 +++ 5 files changed, 80 insertions(+), 67 deletions(-) create mode 100644 documentation/generating_codes.md diff --git a/common/utils/cache/regions.py b/common/utils/cache/regions.py index e2a9724..740317f 100644 --- a/common/utils/cache/regions.py +++ b/common/utils/cache/regions.py @@ -42,24 +42,15 @@ def get_redis_config(app): 'password': redis_uri.password }) - # SSL support using Dogpile's built-in mechanism - cert_data = app.config.get('REDIS_CERT_DATA') - if cert_data and redis_uri.scheme == 'rediss': + # SSL support using centralized config + cert_path = app.config.get('REDIS_CA_CERT_PATH') + if cert_path and redis_uri.scheme == 'rediss': import ssl - import tempfile - # Create SSL context ssl_context = ssl.create_default_context() ssl_context.verify_mode = ssl.CERT_REQUIRED - ssl_context.check_hostname = True - - # Write cert to temp file - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as f: - f.write(cert_data) - ssl_cert_path = f.name - - ssl_context.load_verify_locations(ssl_cert_path) - + ssl_context.check_hostname = app.config.get('REDIS_SSL_CHECK_HOSTNAME', True) + ssl_context.load_verify_locations(cert_path) # Add SSL to connection pool kwargs config['connection_pool_class_kwargs']['ssl'] = ssl_context diff --git a/common/utils/celery_utils.py b/common/utils/celery_utils.py index 3a14fc2..08b07cb 100644 --- a/common/utils/celery_utils.py +++ b/common/utils/celery_utils.py @@ -1,7 +1,4 @@ -import atexit -import os import ssl -import tempfile from celery import Celery from kombu import Queue @@ -9,26 +6,6 @@ from werkzeug.local import LocalProxy from redbeat import RedBeatScheduler celery_app = Celery() -_tmp_paths = [] - -def _create_ssl_cert_file(cert_data: str) -> str: - """Create temporary certificate file for Celery SSL""" - if not cert_data: - return None - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as cert_file: - cert_file.write(cert_data) - path = cert_file.name - _tmp_paths.append(path) # track for cleanup - return path - -def _cleanup_tmp(): - for p in _tmp_paths: - try: - os.remove(p) - except Exception: - pass - -atexit.register(_cleanup_tmp) def init_celery(celery, app, is_beat=False): @@ -67,25 +44,20 @@ def init_celery(celery, app, is_beat=False): # celery_config['result_backend_transport_options'] = result_backend_transport_options # TLS (only when cert is provided or your URLs are rediss://) - cert_data = app.config.get('REDIS_CERT_DATA') ssl_opts = None - if cert_data: - try: - ca_path = _create_ssl_cert_file(cert_data) - if ca_path: - ssl_opts = { - 'ssl_cert_reqs': ssl.CERT_REQUIRED, # <— constant, not string - 'ssl_ca_certs': ca_path, - # 'ssl_check_hostname': True, # kombu/redis doesn’t consistently honor this; CERT_REQUIRED is the key - } - app.logger.info("SSL configured for Celery Redis connection (CA provided)") - except Exception as e: - app.logger.error(f"Failed to configure SSL for Celery: {e}") - - if ssl_opts is None: - ssl_opts = {'ssl_cert_reqs': ssl.CERT_REQUIRED} + cert_path = app.config.get('REDIS_CA_CERT_PATH') + if cert_path: + ssl_opts = { + 'ssl_cert_reqs': ssl.CERT_REQUIRED, + 'ssl_ca_certs': cert_path, + 'ssl_check_hostname': app.config.get('REDIS_SSL_CHECK_HOSTNAME', True), + } + app.logger.info( + "SSL configured for Celery Redis connection (CA: %s, hostname-check: %s)", + cert_path, + 'enabled' if app.config.get('REDIS_SSL_CHECK_HOSTNAME', True) else 'disabled (IP)' + ) celery_config['broker_use_ssl'] = ssl_opts - # Redis result backend needs its own key: celery_config['redis_backend_use_ssl'] = ssl_opts # Beat/RedBeat diff --git a/config/config.py b/config/config.py index 6e711a4..402302d 100644 --- a/config/config.py +++ b/config/config.py @@ -2,6 +2,9 @@ import os from os import environ, path from datetime import timedelta import redis +import ssl +import tempfile +from ipaddress import ip_address from common.utils.prompt_loader import load_prompt_templates @@ -30,11 +33,38 @@ class Config(object): REDIS_PASS = environ.get('REDIS_PASS') REDIS_CERT_DATA = environ.get('REDIS_CERT') + # Determine if REDIS_URL is an IP; use it to control hostname checking + REDIS_IS_IP = False + try: + ip_address(REDIS_URL) + REDIS_IS_IP = True + except Exception: + REDIS_IS_IP = False + REDIS_SSL_CHECK_HOSTNAME = not REDIS_IS_IP + + # Write CA once to a file, expose path + REDIS_CA_CERT_PATH = None + if REDIS_CERT_DATA: + _tmp = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') + _tmp.write(REDIS_CERT_DATA) + _tmp.flush() + _tmp.close() + REDIS_CA_CERT_PATH = _tmp.name + if not REDIS_CERT_DATA: # We are in a simple dev/test environment REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}' else: # We are in a scaleway environment, providing name, user and certificate REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}' + # Central SSL options dict for reuse (Celery/Dogpile/etc.) + REDIS_SSL_OPTIONS = None + if REDIS_CERT_DATA and REDIS_CA_CERT_PATH: + REDIS_SSL_OPTIONS = { + 'ssl_cert_reqs': ssl.CERT_REQUIRED, + 'ssl_ca_certs': REDIS_CA_CERT_PATH, + 'ssl_check_hostname': REDIS_SSL_CHECK_HOSTNAME, + } + REDIS_PREFIXES = { 'celery_app': 'celery:app:', 'celery_chat': 'celery:chat:', @@ -62,7 +92,20 @@ class Config(object): SESSION_USE_SIGNER = True PERMANENT_SESSION_LIFETIME = timedelta(minutes=60) SESSION_REFRESH_EACH_REQUEST = True - SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/0') + # Configure SESSION_REDIS with SSL when cert is provided + if REDIS_CERT_DATA and REDIS_CA_CERT_PATH: + SESSION_REDIS = redis.Redis( + host=REDIS_URL, + port=int(REDIS_PORT or 6379), + username=REDIS_USER, + password=REDIS_PASS, + ssl=True, + ssl_cert_reqs=ssl.CERT_REQUIRED, + ssl_ca_certs=REDIS_CA_CERT_PATH, + ssl_check_hostname=REDIS_SSL_CHECK_HOSTNAME, + ) + else: + SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/0') SESSION_KEY_PREFIX = f'session_{COMPONENT_NAME}:' SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session' SESSION_COOKIE_DOMAIN = None # Laat Flask dit automatisch bepalen diff --git a/documentation/Production Setup/cluster-install.md b/documentation/Production Setup/cluster-install.md index c66086a..8c9defe 100644 --- a/documentation/Production Setup/cluster-install.md +++ b/documentation/Production Setup/cluster-install.md @@ -467,20 +467,14 @@ kubectl -n tools port-forward svc/pgadmin-pgadmin4 8080:80 ### Phase 9: Enable Scaleway Registry -1) Create docker pull secret via External Secrets (once): +#### Create Scaleway Registry Secret +Create docker pull secret via External Secrets (once): ```bash kubectl apply -f scaleway/manifests/base/secrets/scaleway-registry-secret.yaml kubectl -n eveai-staging get secret scaleway-registry-cred -o yaml | grep "type: kubernetes.io/dockerconfigjson" ``` -2) Use the staging overlay to deploy apps with registry rewrite and imagePullSecrets: -```bash -kubectl apply -k scaleway/manifests/overlays/staging/ -``` -Notes: -- Base manifests keep generic images (josakola/...). The overlay rewrites them to rg.fr-par.scw.cloud/eveai-staging/josakola/...:staging and adds imagePullSecrets to all Pods. -- Staging uses imagePullPolicy: Always, so new pushes to :staging are pulled automatically. -### Phase 10: Ops Jobs Invocation (if required) +#### Ops Jobs Invocation (if required) Run the DB ops scripts manually in order. Each manifest uses generateName; use kubectl create. @@ -510,9 +504,14 @@ kubectl -n eveai-staging get jobs kubectl -n eveai-staging logs job/ ``` -### Phase 11: Application Services Deployment - - +#### Application Services Deployment +Use the staging overlay to deploy apps with registry rewrite and imagePullSecrets: +```bash +kubectl apply -k scaleway/manifests/overlays/staging/ +``` +Notes: +- Base manifests keep generic images (josakola/...). The overlay rewrites them to rg.fr-par.scw.cloud/eveai-staging/josakola/...:staging and adds imagePullSecrets to all Pods. +- Staging uses imagePullPolicy: Always, so new pushes to :staging are pulled automatically. ## Verification and Testing diff --git a/documentation/generating_codes.md b/documentation/generating_codes.md new file mode 100644 index 0000000..c212852 --- /dev/null +++ b/documentation/generating_codes.md @@ -0,0 +1,8 @@ +# Generating codes + +## API_ENCRYPTION_KEY + +```python +from cryptography.fernet import Fernet; +print(Fernet.generate_key().decode()) +``` \ No newline at end of file