From e6c3c24bd80223647b8aa7f3afbfa83f452a9184 Mon Sep 17 00:00:00 2001 From: Josako Date: Fri, 22 Aug 2025 10:47:03 +0200 Subject: [PATCH] - In Scaleway, we only have one bucket, and store information for each tenant in separate folders - Added staging configuration to scaleway --- common/utils/minio_utils.py | 56 +++++++++++++------ config/config.py | 92 ++++++++++++++++++------------- eveai_app/__init__.py | 2 + eveai_app/views/document_views.py | 5 +- 4 files changed, 99 insertions(+), 56 deletions(-) diff --git a/common/utils/minio_utils.py b/common/utils/minio_utils.py index c5e2a75..f950532 100644 --- a/common/utils/minio_utils.py +++ b/common/utils/minio_utils.py @@ -1,6 +1,6 @@ from minio import Minio from minio.error import S3Error -from flask import Flask +from flask import Flask, current_app import io from werkzeug.datastructures import FileStorage @@ -21,27 +21,51 @@ class MinioClient: app.logger.info(f"MinIO client initialized with endpoint: {app.config['MINIO_ENDPOINT']}") def generate_bucket_name(self, tenant_id): - return f"tenant-{tenant_id}-bucket" + tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket') + if tenant_base == 'bucket': + return f"tenant-{tenant_id}-bucket" + elif tenant_base == 'folder': + return current_app.config.get('OBJECT_STORAGE_BUCKET_NAME') + else: + raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}") def create_tenant_bucket(self, tenant_id): - bucket_name = self.generate_bucket_name(tenant_id) - try: - if not self.client.bucket_exists(bucket_name): - self.client.make_bucket(bucket_name) + tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket') + if tenant_base == 'bucket': + bucket_name = self.generate_bucket_name(tenant_id) + try: + if not self.client.bucket_exists(bucket_name): + self.client.make_bucket(bucket_name) + return bucket_name return bucket_name - return bucket_name - except S3Error as err: - raise Exception(f"Error occurred while creating bucket: {err}") + except S3Error as err: + raise Exception(f"Error occurred while creating bucket: {err}") + elif tenant_base == 'folder': # In this case, we are working within a predefined bucket + return current_app.config.get('OBJECT_STORAGE_BUCKET_NAME') + else: + raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}") - def generate_object_name(self, document_id, language, version_id, filename): - return f"{document_id}/{language}/{version_id}/{filename}" + def generate_object_name(self, tenant_id, document_id, language, version_id, filename): + tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket') + if tenant_base == 'bucket': + return f"{document_id}/{language}/{version_id}/{filename}" + elif tenant_base == 'folder': + return f"tenant-{tenant_id}/documents/{document_id}/{language}/{version_id}/{filename}" + else: + raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}") - def generate_asset_name(self, asset_id, asset_type, content_type): - return f"assets/{asset_type}/{asset_id}.{content_type}" + def generate_asset_name(self, tenant_id, asset_id, asset_type, content_type): + tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket') + if tenant_base == 'bucket': + return f"assets/{asset_type}/{asset_id}.{content_type}" + elif tenant_base == 'folder': + return f"tenant-{tenant_id}/assets/{asset_type}/{asset_id}.{content_type}" + else: + raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}") def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data): bucket_name = self.generate_bucket_name(tenant_id) - object_name = self.generate_object_name(document_id, language, version_id, filename) + object_name = self.generate_object_name(tenant_id, document_id, language, version_id, filename) try: if isinstance(file_data, FileStorage): @@ -63,7 +87,7 @@ class MinioClient: def upload_asset_file(self, tenant_id: int, asset_id: int, asset_type: str, file_type: str, file_data: bytes | FileStorage | io.BytesIO | str, ) -> tuple[str, str, int]: bucket_name = self.generate_bucket_name(tenant_id) - object_name = self.generate_asset_name(asset_id, asset_type, file_type) + object_name = self.generate_asset_name(tenant_id, asset_id, asset_type, file_type) try: if isinstance(file_data, FileStorage): @@ -111,7 +135,7 @@ class MinioClient: def delete_document_file(self, tenant_id, document_id, language, version_id, filename): bucket_name = self.generate_bucket_name(tenant_id) - object_name = self.generate_object_name(document_id, language, version_id, filename) + object_name = self.generate_object_name(tenant_id, document_id, language, version_id, filename) try: self.client.remove_object(bucket_name, object_name) return True diff --git a/config/config.py b/config/config.py index 72f2d16..de2e87b 100644 --- a/config/config.py +++ b/config/config.py @@ -298,27 +298,54 @@ class DevConfig(Config): # eveai_model cache Redis setting MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6' + # Session settings + SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') - # Unstructured settings - # UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw' - # UNSTRUCTURED_BASE_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io' - # UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general' + # PATH settings + ffmpeg_path = '/usr/bin/ffmpeg' - # SocketIO settings - # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1' - # SOCKETIO_CORS_ALLOWED_ORIGINS = '*' - # SOCKETIO_LOGGER = True - # SOCKETIO_ENGINEIO_LOGGER = True - # SOCKETIO_PING_TIMEOUT = 20000 - # SOCKETIO_PING_INTERVAL = 25000 - # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60) # Changing this value ==> change maxConnectionDuration value in - # eveai-chat-widget.js + # OBJECT STORAGE + OBJECT_STORAGE_TYPE = 'MINIO' + OBJECT_STORAGE_TENANT_BASE = 'Bucket' + # MINIO + MINIO_ENDPOINT = 'minio:9000' + MINIO_ACCESS_KEY = 'minioadmin' + MINIO_SECRET_KEY = 'minioadmin' + MINIO_USE_HTTPS = False - # Google Cloud settings - GC_PROJECT_NAME = 'eveai-420711' - GC_LOCATION = 'europe-west1' - GC_KEY_RING = 'eveai-chat' - GC_CRYPTO_KEY = 'envelope-encryption-key' + +class StagingConfig(Config): + DEVELOPMENT = False + DEBUG = True + FLASK_DEBUG = True + EXPLAIN_TEMPLATE_LOADING = False + + # Define the nginx prefix used for the specific apps + EVEAI_APP_LOCATION_PREFIX = '/admin' + EVEAI_CHAT_LOCATION_PREFIX = '/chat' + CHAT_CLIENT_PREFIX = 'chat-client/chat/' + + # file upload settings + # UPLOAD_FOLDER = '/app/tenant_files' + + # Redis Settings + REDIS_URL = 'redis' + REDIS_PORT = '6379' + REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}' + + # Celery settings + # eveai_app Redis Settings + CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0' + CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0' + # eveai_chat Redis Settings + CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' + CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' + # eveai_chat_workers cache Redis Settings + CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4' + # specialist execution pub/sub Redis Settings + SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5' + # eveai_model cache Redis setting + MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6' # Session settings SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') @@ -326,11 +353,15 @@ class DevConfig(Config): # PATH settings ffmpeg_path = '/usr/bin/ffmpeg' + # OBJECT STORAGE + OBJECT_STORAGE_TYPE = 'SCALEWAY' + OBJECT_STORAGE_TENANT_BASE = 'Folder' + OBJECT_STORAGE_BUCKET_NAME = 'eveai-staging' # MINIO - MINIO_ENDPOINT = 'minio:9000' - MINIO_ACCESS_KEY = 'minioadmin' - MINIO_SECRET_KEY = 'minioadmin' - MINIO_USE_HTTPS = False + MINIO_ENDPOINT = 'https://eveai-staging.s3.fr-par.scw.cloud' + MINIO_ACCESS_KEY = environ.get('SCALEWAY_EVEAI_STAGING_ACCESS_KEY') + MINIO_SECRET_KEY = environ.get('SCALEWAY_EVEAI_STAGING_SECRET_KEY') + MINIO_USE_HTTPS = True class ProdConfig(Config): @@ -377,22 +408,6 @@ class ProdConfig(Config): # Session settings SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') - # SocketIO settings - # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1' - # SOCKETIO_CORS_ALLOWED_ORIGINS = '*' - # SOCKETIO_LOGGER = True - # SOCKETIO_ENGINEIO_LOGGER = True - # SOCKETIO_PING_TIMEOUT = 20000 - # SOCKETIO_PING_INTERVAL = 25000 - # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60) # Changing this value ==> change maxConnectionDuration value in - # eveai-chat-widget.js - - # Google Cloud settings - GC_PROJECT_NAME = 'eveai-420711' - GC_LOCATION = 'europe-west1' - GC_KEY_RING = 'eveai-chat' - GC_CRYPTO_KEY = 'envelope-encryption-key' - # PATH settings ffmpeg_path = '/usr/bin/ffmpeg' @@ -406,6 +421,7 @@ class ProdConfig(Config): def get_config(config_name='dev'): configs = { 'dev': DevConfig, + 'staging': StagingConfig, 'prod': ProdConfig, 'default': DevConfig, } diff --git a/eveai_app/__init__.py b/eveai_app/__init__.py index 7258713..3b87385 100644 --- a/eveai_app/__init__.py +++ b/eveai_app/__init__.py @@ -33,6 +33,8 @@ def create_app(config_file=None): match environment: case 'development': app.config.from_object(get_config('dev')) + case 'staging': + app.config.from_object(get_config('staging')) case 'production': app.config.from_object(get_config('prod')) case _: diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index bf5a5cc..56ced27 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -622,8 +622,9 @@ def view_document_version_markdown(document_version_id): try: # Generate markdown filename markdown_filename = f"{document_version.id}.md" - markdown_object_name = minio_client.generate_object_name(document_version.doc_id, document_version.language, - document_version.id, markdown_filename) + markdown_object_name = minio_client.generate_object_name(tenant_id, + document_version.doc_id, document_version.language, + document_version.id, markdown_filename) # Download actual markdown file file_data = minio_client.download_document_file( tenant_id,