- In Scaleway, we only have one bucket, and store information for each tenant in separate folders

- Added staging configuration to scaleway
2025-08-22 10:47:03 +02:00
parent 376ad328ca
commit e6c3c24bd8
4 changed files with 99 additions and 56 deletions
--- a/common/utils/minio_utils.py
+++ b/common/utils/minio_utils.py
@@ -1,6 +1,6 @@
 from minio import Minio
 from minio.error import S3Error
-from flask import Flask
+from flask import Flask, current_app
 import io
 from werkzeug.datastructures import FileStorage

@@ -21,9 +21,17 @@ class MinioClient:
        app.logger.info(f"MinIO client initialized with endpoint: {app.config['MINIO_ENDPOINT']}")

    def generate_bucket_name(self, tenant_id):
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
            return f"tenant-{tenant_id}-bucket"
+        elif tenant_base == 'folder':
+            return current_app.config.get('OBJECT_STORAGE_BUCKET_NAME')
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")

    def create_tenant_bucket(self, tenant_id):
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
            bucket_name = self.generate_bucket_name(tenant_id)
            try:
                if not self.client.bucket_exists(bucket_name):
@@ -32,16 +40,32 @@ class MinioClient:
                return bucket_name
            except S3Error as err:
                raise Exception(f"Error occurred while creating bucket: {err}")
+        elif tenant_base == 'folder':   # In this case, we are working within a predefined bucket
+            return current_app.config.get('OBJECT_STORAGE_BUCKET_NAME')
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")

-    def generate_object_name(self, document_id, language, version_id, filename):
+    def generate_object_name(self, tenant_id, document_id, language, version_id, filename):
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
            return f"{document_id}/{language}/{version_id}/{filename}"
+        elif tenant_base == 'folder':
+            return f"tenant-{tenant_id}/documents/{document_id}/{language}/{version_id}/{filename}"
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")

-    def generate_asset_name(self, asset_id, asset_type, content_type):
+    def generate_asset_name(self, tenant_id, asset_id, asset_type, content_type):
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
            return f"assets/{asset_type}/{asset_id}.{content_type}"
+        elif tenant_base == 'folder':
+            return f"tenant-{tenant_id}/assets/{asset_type}/{asset_id}.{content_type}"
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")

    def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data):
        bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_object_name(document_id, language, version_id, filename)
+        object_name = self.generate_object_name(tenant_id, document_id, language, version_id, filename)

        try:
            if isinstance(file_data, FileStorage):
@@ -63,7 +87,7 @@ class MinioClient:
    def upload_asset_file(self, tenant_id: int, asset_id: int, asset_type: str, file_type: str,
                          file_data: bytes | FileStorage | io.BytesIO | str, ) -> tuple[str, str, int]:
        bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_asset_name(asset_id, asset_type, file_type)
+        object_name = self.generate_asset_name(tenant_id, asset_id, asset_type, file_type)

        try:
            if isinstance(file_data, FileStorage):
@@ -111,7 +135,7 @@ class MinioClient:

    def delete_document_file(self, tenant_id, document_id, language, version_id, filename):
        bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_object_name(document_id, language, version_id, filename)
+        object_name = self.generate_object_name(tenant_id, document_id, language, version_id, filename)
        try:
            self.client.remove_object(bucket_name, object_name)
            return True
--- a/config/config.py
+++ b/config/config.py
@@ -298,27 +298,54 @@ class DevConfig(Config):
    # eveai_model cache Redis setting
    MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'

+    # Session settings
+    SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')

-    # Unstructured settings
-    # UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw'
-    # UNSTRUCTURED_BASE_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io'
-    # UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general'
+    # PATH settings
+    ffmpeg_path = '/usr/bin/ffmpeg'

-    # SocketIO settings
-    # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
-    # SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
-    # SOCKETIO_LOGGER = True
-    # SOCKETIO_ENGINEIO_LOGGER = True
-    # SOCKETIO_PING_TIMEOUT = 20000
-    # SOCKETIO_PING_INTERVAL = 25000
-    # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
-                                                    # eveai-chat-widget.js
+    # OBJECT STORAGE
+    OBJECT_STORAGE_TYPE = 'MINIO'
+    OBJECT_STORAGE_TENANT_BASE = 'Bucket'
+    # MINIO
+    MINIO_ENDPOINT = 'minio:9000'
+    MINIO_ACCESS_KEY = 'minioadmin'
+    MINIO_SECRET_KEY = 'minioadmin'
+    MINIO_USE_HTTPS = False

-    # Google Cloud settings
-    GC_PROJECT_NAME = 'eveai-420711'
-    GC_LOCATION = 'europe-west1'
-    GC_KEY_RING = 'eveai-chat'
-    GC_CRYPTO_KEY = 'envelope-encryption-key'
+
+class StagingConfig(Config):
+    DEVELOPMENT = False
+    DEBUG = True
+    FLASK_DEBUG = True
+    EXPLAIN_TEMPLATE_LOADING = False
+
+    # Define the nginx prefix used for the specific apps
+    EVEAI_APP_LOCATION_PREFIX = '/admin'
+    EVEAI_CHAT_LOCATION_PREFIX = '/chat'
+    CHAT_CLIENT_PREFIX = 'chat-client/chat/'
+
+    # file upload settings
+    # UPLOAD_FOLDER = '/app/tenant_files'
+
+    # Redis Settings
+    REDIS_URL = 'redis'
+    REDIS_PORT = '6379'
+    REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
+
+    # Celery settings
+    # eveai_app Redis Settings
+    CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
+    CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
+    # eveai_chat Redis Settings
+    CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
+    CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
+    # eveai_chat_workers cache Redis Settings
+    CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
+    # specialist execution pub/sub Redis Settings
+    SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
+    # eveai_model cache Redis setting
+    MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'

    # Session settings
    SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
@@ -326,11 +353,15 @@ class DevConfig(Config):
    # PATH settings
    ffmpeg_path = '/usr/bin/ffmpeg'

+    # OBJECT STORAGE
+    OBJECT_STORAGE_TYPE = 'SCALEWAY'
+    OBJECT_STORAGE_TENANT_BASE = 'Folder'
+    OBJECT_STORAGE_BUCKET_NAME = 'eveai-staging'
    # MINIO
-    MINIO_ENDPOINT = 'minio:9000'
-    MINIO_ACCESS_KEY = 'minioadmin'
-    MINIO_SECRET_KEY = 'minioadmin'
-    MINIO_USE_HTTPS = False
+    MINIO_ENDPOINT = 'https://eveai-staging.s3.fr-par.scw.cloud'
+    MINIO_ACCESS_KEY = environ.get('SCALEWAY_EVEAI_STAGING_ACCESS_KEY')
+    MINIO_SECRET_KEY = environ.get('SCALEWAY_EVEAI_STAGING_SECRET_KEY')
+    MINIO_USE_HTTPS = True


 class ProdConfig(Config):
@@ -377,22 +408,6 @@ class ProdConfig(Config):
    # Session settings
    SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')

-    # SocketIO settings
-    # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
-    # SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
-    # SOCKETIO_LOGGER = True
-    # SOCKETIO_ENGINEIO_LOGGER = True
-    # SOCKETIO_PING_TIMEOUT = 20000
-    # SOCKETIO_PING_INTERVAL = 25000
-    # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
-    # eveai-chat-widget.js
-
-    # Google Cloud settings
-    GC_PROJECT_NAME = 'eveai-420711'
-    GC_LOCATION = 'europe-west1'
-    GC_KEY_RING = 'eveai-chat'
-    GC_CRYPTO_KEY = 'envelope-encryption-key'
-
    # PATH settings
    ffmpeg_path = '/usr/bin/ffmpeg'

@@ -406,6 +421,7 @@ class ProdConfig(Config):
 def get_config(config_name='dev'):
    configs = {
        'dev': DevConfig,
+        'staging': StagingConfig,
        'prod': ProdConfig,
        'default': DevConfig,
    }
--- a/eveai_app/init.py
+++ b/eveai_app/init.py
@@ -33,6 +33,8 @@ def create_app(config_file=None):
    match environment:
        case 'development':
            app.config.from_object(get_config('dev'))
+        case 'staging':
+            app.config.from_object(get_config('staging'))
        case 'production':
            app.config.from_object(get_config('prod'))
        case _:
--- a/eveai_app/views/document_views.py
+++ b/eveai_app/views/document_views.py
@@ -622,7 +622,8 @@ def view_document_version_markdown(document_version_id):
    try:
        # Generate markdown filename
        markdown_filename = f"{document_version.id}.md"
-        markdown_object_name = minio_client.generate_object_name(document_version.doc_id, document_version.language,
+        markdown_object_name = minio_client.generate_object_name(tenant_id,
+            document_version.doc_id, document_version.language,
            document_version.id, markdown_filename)
        # Download actual markdown file
        file_data = minio_client.download_document_file(