pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- TLS Refactoring

Browse Source
This commit is contained in:
Josako
2025-09-04 15:22:45 +02:00
parent af8b5f54cd
commit 54a9641440
5 changed files with 80 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
common/utils/cache/regions.py vendored
View File

@@ -42,24 +42,15 @@ def get_redis_config(app):
'password': redis_uri.password
})
# SSL support using Dogpile's built-in mechanism
cert_data = app.config.get('REDIS_CERT_DATA')
if cert_data and redis_uri.scheme == 'rediss':
# SSL support using centralized config
cert_path = app.config.get('REDIS_CA_CERT_PATH')
if cert_path and redis_uri.scheme == 'rediss':
import ssl
import tempfile
# Create SSL context
ssl_context = ssl.create_default_context()
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.check_hostname = True
# Write cert to temp file
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as f:
f.write(cert_data)
ssl_cert_path = f.name
ssl_context.load_verify_locations(ssl_cert_path)
ssl_context.check_hostname = app.config.get('REDIS_SSL_CHECK_HOSTNAME', True)
ssl_context.load_verify_locations(cert_path)
# Add SSL to connection pool kwargs
config['connection_pool_class_kwargs']['ssl'] = ssl_context

48
common/utils/celery_utils.py
View File

@@ -1,7 +1,4 @@
import atexit
import os
import ssl
import tempfile
from celery import Celery
from kombu import Queue
@@ -9,26 +6,6 @@ from werkzeug.local import LocalProxy
from redbeat import RedBeatScheduler
celery_app = Celery()
_tmp_paths = []
def _create_ssl_cert_file(cert_data: str) -> str:
"""Create temporary certificate file for Celery SSL"""
if not cert_data:
return None
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') as cert_file:
cert_file.write(cert_data)
path = cert_file.name
_tmp_paths.append(path) # track for cleanup
return path
def _cleanup_tmp():
for p in _tmp_paths:
try:
os.remove(p)
except Exception:
pass
atexit.register(_cleanup_tmp)
def init_celery(celery, app, is_beat=False):
@@ -67,25 +44,20 @@ def init_celery(celery, app, is_beat=False):
# celery_config['result_backend_transport_options'] = result_backend_transport_options
# TLS (only when cert is provided or your URLs are rediss://)
cert_data = app.config.get('REDIS_CERT_DATA')
ssl_opts = None
if cert_data:
try:
ca_path = _create_ssl_cert_file(cert_data)
if ca_path:
cert_path = app.config.get('REDIS_CA_CERT_PATH')
if cert_path:
ssl_opts = {
'ssl_cert_reqs': ssl.CERT_REQUIRED, # <— constant, not string
'ssl_ca_certs': ca_path,
# 'ssl_check_hostname': True, # kombu/redis doesn’t consistently honor this; CERT_REQUIRED is the key
'ssl_cert_reqs': ssl.CERT_REQUIRED,
'ssl_ca_certs': cert_path,
'ssl_check_hostname': app.config.get('REDIS_SSL_CHECK_HOSTNAME', True),
}
app.logger.info("SSL configured for Celery Redis connection (CA provided)")
except Exception as e:
app.logger.error(f"Failed to configure SSL for Celery: {e}")
if ssl_opts is None:
ssl_opts = {'ssl_cert_reqs': ssl.CERT_REQUIRED}
app.logger.info(
"SSL configured for Celery Redis connection (CA: %s, hostname-check: %s)",
cert_path,
'enabled' if app.config.get('REDIS_SSL_CHECK_HOSTNAME', True) else 'disabled (IP)'
)
celery_config['broker_use_ssl'] = ssl_opts
# Redis result backend needs its own key:
celery_config['redis_backend_use_ssl'] = ssl_opts
# Beat/RedBeat

43
config/config.py
View File

@@ -2,6 +2,9 @@ import os
from os import environ, path
from datetime import timedelta
import redis
import ssl
import tempfile
from ipaddress import ip_address
from common.utils.prompt_loader import load_prompt_templates
@@ -30,11 +33,38 @@ class Config(object):
REDIS_PASS = environ.get('REDIS_PASS')
REDIS_CERT_DATA = environ.get('REDIS_CERT')
# Determine if REDIS_URL is an IP; use it to control hostname checking
REDIS_IS_IP = False
try:
ip_address(REDIS_URL)
REDIS_IS_IP = True
except Exception:
REDIS_IS_IP = False
REDIS_SSL_CHECK_HOSTNAME = not REDIS_IS_IP
# Write CA once to a file, expose path
REDIS_CA_CERT_PATH = None
if REDIS_CERT_DATA:
_tmp = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem')
_tmp.write(REDIS_CERT_DATA)
_tmp.flush()
_tmp.close()
REDIS_CA_CERT_PATH = _tmp.name
if not REDIS_CERT_DATA: # We are in a simple dev/test environment
REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
else: # We are in a scaleway environment, providing name, user and certificate
REDIS_BASE_URI = f'rediss://{REDIS_USER}:{REDIS_PASS}@{REDIS_URL}:{REDIS_PORT}'
# Central SSL options dict for reuse (Celery/Dogpile/etc.)
REDIS_SSL_OPTIONS = None
if REDIS_CERT_DATA and REDIS_CA_CERT_PATH:
REDIS_SSL_OPTIONS = {
'ssl_cert_reqs': ssl.CERT_REQUIRED,
'ssl_ca_certs': REDIS_CA_CERT_PATH,
'ssl_check_hostname': REDIS_SSL_CHECK_HOSTNAME,
}
REDIS_PREFIXES = {
'celery_app': 'celery:app:',
'celery_chat': 'celery:chat:',
@@ -62,6 +92,19 @@ class Config(object):
SESSION_USE_SIGNER = True
PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
SESSION_REFRESH_EACH_REQUEST = True
# Configure SESSION_REDIS with SSL when cert is provided
if REDIS_CERT_DATA and REDIS_CA_CERT_PATH:
SESSION_REDIS = redis.Redis(
host=REDIS_URL,
port=int(REDIS_PORT or 6379),
username=REDIS_USER,
password=REDIS_PASS,
ssl=True,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=REDIS_CA_CERT_PATH,
ssl_check_hostname=REDIS_SSL_CHECK_HOSTNAME,
)
else:
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/0')
SESSION_KEY_PREFIX = f'session_{COMPONENT_NAME}:'
SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session'

23
documentation/Production Setup/cluster-install.md
View File

@@ -467,20 +467,14 @@ kubectl -n tools port-forward svc/pgadmin-pgadmin4 8080:80
### Phase 9: Enable Scaleway Registry
1) Create docker pull secret via External Secrets (once):
#### Create Scaleway Registry Secret
Create docker pull secret via External Secrets (once):
```bash
kubectl apply -f scaleway/manifests/base/secrets/scaleway-registry-secret.yaml
kubectl -n eveai-staging get secret scaleway-registry-cred -o yaml | grep "type: kubernetes.io/dockerconfigjson"
```
2) Use the staging overlay to deploy apps with registry rewrite and imagePullSecrets:
```bash
kubectl apply -k scaleway/manifests/overlays/staging/
```
Notes:
- Base manifests keep generic images (josakola/...). The overlay rewrites them to rg.fr-par.scw.cloud/eveai-staging/josakola/...:staging and adds imagePullSecrets to all Pods.
- Staging uses imagePullPolicy: Always, so new pushes to :staging are pulled automatically.
### Phase 10: Ops Jobs Invocation (if required)
#### Ops Jobs Invocation (if required)
Run the DB ops scripts manually in order. Each manifest uses generateName; use kubectl create.
@@ -510,9 +504,14 @@ kubectl -n eveai-staging get jobs
kubectl -n eveai-staging logs job/<created-job-name>
```
### Phase 11: Application Services Deployment
#### Application Services Deployment
Use the staging overlay to deploy apps with registry rewrite and imagePullSecrets:
```bash
kubectl apply -k scaleway/manifests/overlays/staging/
```
Notes:
- Base manifests keep generic images (josakola/...). The overlay rewrites them to rg.fr-par.scw.cloud/eveai-staging/josakola/...:staging and adds imagePullSecrets to all Pods.
- Staging uses imagePullPolicy: Always, so new pushes to :staging are pulled automatically.
## Verification and Testing

8
documentation/generating_codes.md Normal file
View File

@@ -0,0 +1,8 @@
# Generating codes
## API_ENCRYPTION_KEY
```python
from cryptography.fernet import Fernet;
print(Fernet.generate_key().decode())
```