- In Scaleway, we only have one bucket, and store information for each tenant in separate folders

- Added staging configuration to scaleway

common/utils/minio_utils.py

@@ -1,6 +1,6 @@
 from minio import Minio
 from minio.error import S3Error
-from flask import Flask
+from flask import Flask, current_app
 import io
 from werkzeug.datastructures import FileStorage
 
@@ -21,27 +21,51 @@ class MinioClient:
         app.logger.info(f"MinIO client initialized with endpoint: {app.config['MINIO_ENDPOINT']}")
 
     def generate_bucket_name(self, tenant_id):
-        return f"tenant-{tenant_id}-bucket"
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
+            return f"tenant-{tenant_id}-bucket"
+        elif tenant_base == 'folder':
+            return current_app.config.get('OBJECT_STORAGE_BUCKET_NAME')
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")
 
     def create_tenant_bucket(self, tenant_id):
-        bucket_name = self.generate_bucket_name(tenant_id)
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
-        try:
+        if tenant_base == 'bucket':
-            if not self.client.bucket_exists(bucket_name):
+            bucket_name = self.generate_bucket_name(tenant_id)
-                self.client.make_bucket(bucket_name)
+            try:
+                if not self.client.bucket_exists(bucket_name):
+                    self.client.make_bucket(bucket_name)
+                    return bucket_name
                 return bucket_name
-            return bucket_name
+            except S3Error as err:
-        except S3Error as err:
+                raise Exception(f"Error occurred while creating bucket: {err}")
-            raise Exception(f"Error occurred while creating bucket: {err}")
+        elif tenant_base == 'folder':   # In this case, we are working within a predefined bucket
+            return current_app.config.get('OBJECT_STORAGE_BUCKET_NAME')
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")
 
-    def generate_object_name(self, document_id, language, version_id, filename):
+    def generate_object_name(self, tenant_id, document_id, language, version_id, filename):
-        return f"{document_id}/{language}/{version_id}/{filename}"
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
+            return f"{document_id}/{language}/{version_id}/{filename}"
+        elif tenant_base == 'folder':
+            return f"tenant-{tenant_id}/documents/{document_id}/{language}/{version_id}/{filename}"
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")
 
-    def generate_asset_name(self, asset_id, asset_type, content_type):
+    def generate_asset_name(self, tenant_id, asset_id, asset_type, content_type):
-        return f"assets/{asset_type}/{asset_id}.{content_type}"
+        tenant_base = current_app.config.get('OBJECT_STORAGE_TENANT_BASE', 'bucket')
+        if tenant_base == 'bucket':
+            return f"assets/{asset_type}/{asset_id}.{content_type}"
+        elif tenant_base == 'folder':
+            return f"tenant-{tenant_id}/assets/{asset_type}/{asset_id}.{content_type}"
+        else:
+            raise ValueError(f"Invalid OBJECT_STORAGE_TENANT_BASE value: {tenant_base}")
 
     def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data):
         bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_object_name(document_id, language, version_id, filename)
+        object_name = self.generate_object_name(tenant_id, document_id, language, version_id, filename)
 
         try:
             if isinstance(file_data, FileStorage):
@@ -63,7 +87,7 @@ class MinioClient:
     def upload_asset_file(self, tenant_id: int, asset_id: int, asset_type: str, file_type: str,
                           file_data: bytes | FileStorage | io.BytesIO | str, ) -> tuple[str, str, int]:
         bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_asset_name(asset_id, asset_type, file_type)
+        object_name = self.generate_asset_name(tenant_id, asset_id, asset_type, file_type)
 
         try:
             if isinstance(file_data, FileStorage):
@@ -111,7 +135,7 @@ class MinioClient:
 
     def delete_document_file(self, tenant_id, document_id, language, version_id, filename):
         bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_object_name(document_id, language, version_id, filename)
+        object_name = self.generate_object_name(tenant_id, document_id, language, version_id, filename)
         try:
             self.client.remove_object(bucket_name, object_name)
             return True

config/config.py

@@ -298,27 +298,54 @@ class DevConfig(Config):
     # eveai_model cache Redis setting
     MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'
 
+    # Session settings
+    SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
 
-    # Unstructured settings
+    # PATH settings
-    # UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw'
+    ffmpeg_path = '/usr/bin/ffmpeg'
-    # UNSTRUCTURED_BASE_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io'
-    # UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general'
 
-    # SocketIO settings
+    # OBJECT STORAGE
-    # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
+    OBJECT_STORAGE_TYPE = 'MINIO'
-    # SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
+    OBJECT_STORAGE_TENANT_BASE = 'Bucket'
-    # SOCKETIO_LOGGER = True
+    # MINIO
-    # SOCKETIO_ENGINEIO_LOGGER = True
+    MINIO_ENDPOINT = 'minio:9000'
-    # SOCKETIO_PING_TIMEOUT = 20000
+    MINIO_ACCESS_KEY = 'minioadmin'
-    # SOCKETIO_PING_INTERVAL = 25000
+    MINIO_SECRET_KEY = 'minioadmin'
-    # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
+    MINIO_USE_HTTPS = False
-                                                    # eveai-chat-widget.js
 
-    # Google Cloud settings
+
-    GC_PROJECT_NAME = 'eveai-420711'
+class StagingConfig(Config):
-    GC_LOCATION = 'europe-west1'
+    DEVELOPMENT = False
-    GC_KEY_RING = 'eveai-chat'
+    DEBUG = True
-    GC_CRYPTO_KEY = 'envelope-encryption-key'
+    FLASK_DEBUG = True
+    EXPLAIN_TEMPLATE_LOADING = False
+
+    # Define the nginx prefix used for the specific apps
+    EVEAI_APP_LOCATION_PREFIX = '/admin'
+    EVEAI_CHAT_LOCATION_PREFIX = '/chat'
+    CHAT_CLIENT_PREFIX = 'chat-client/chat/'
+
+    # file upload settings
+    # UPLOAD_FOLDER = '/app/tenant_files'
+
+    # Redis Settings
+    REDIS_URL = 'redis'
+    REDIS_PORT = '6379'
+    REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
+
+    # Celery settings
+    # eveai_app Redis Settings
+    CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
+    CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
+    # eveai_chat Redis Settings
+    CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
+    CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
+    # eveai_chat_workers cache Redis Settings
+    CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
+    # specialist execution pub/sub Redis Settings
+    SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
+    # eveai_model cache Redis setting
+    MODEL_CACHE_URL = f'{REDIS_BASE_URI}/6'
 
     # Session settings
     SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
@@ -326,11 +353,15 @@ class DevConfig(Config):
     # PATH settings
     ffmpeg_path = '/usr/bin/ffmpeg'
 
+    # OBJECT STORAGE
+    OBJECT_STORAGE_TYPE = 'SCALEWAY'
+    OBJECT_STORAGE_TENANT_BASE = 'Folder'
+    OBJECT_STORAGE_BUCKET_NAME = 'eveai-staging'
     # MINIO
-    MINIO_ENDPOINT = 'minio:9000'
+    MINIO_ENDPOINT = 'https://eveai-staging.s3.fr-par.scw.cloud'
-    MINIO_ACCESS_KEY = 'minioadmin'
+    MINIO_ACCESS_KEY = environ.get('SCALEWAY_EVEAI_STAGING_ACCESS_KEY')
-    MINIO_SECRET_KEY = 'minioadmin'
+    MINIO_SECRET_KEY = environ.get('SCALEWAY_EVEAI_STAGING_SECRET_KEY')
-    MINIO_USE_HTTPS = False
+    MINIO_USE_HTTPS = True
 
 
 class ProdConfig(Config):
@@ -377,22 +408,6 @@ class ProdConfig(Config):
     # Session settings
     SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
 
-    # SocketIO settings
-    # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
-    # SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
-    # SOCKETIO_LOGGER = True
-    # SOCKETIO_ENGINEIO_LOGGER = True
-    # SOCKETIO_PING_TIMEOUT = 20000
-    # SOCKETIO_PING_INTERVAL = 25000
-    # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
-    # eveai-chat-widget.js
-
-    # Google Cloud settings
-    GC_PROJECT_NAME = 'eveai-420711'
-    GC_LOCATION = 'europe-west1'
-    GC_KEY_RING = 'eveai-chat'
-    GC_CRYPTO_KEY = 'envelope-encryption-key'
-
     # PATH settings
     ffmpeg_path = '/usr/bin/ffmpeg'
 
@@ -406,6 +421,7 @@ class ProdConfig(Config):
 def get_config(config_name='dev'):
     configs = {
         'dev': DevConfig,
+        'staging': StagingConfig,
         'prod': ProdConfig,
         'default': DevConfig,
     }

eveai_app/__init__.py

@@ -33,6 +33,8 @@ def create_app(config_file=None):
     match environment:
         case 'development':
             app.config.from_object(get_config('dev'))
+        case 'staging':
+            app.config.from_object(get_config('staging'))
         case 'production':
             app.config.from_object(get_config('prod'))
         case _:

eveai_app/views/document_views.py

@@ -622,8 +622,9 @@ def view_document_version_markdown(document_version_id):
     try:
         # Generate markdown filename
         markdown_filename = f"{document_version.id}.md"
-        markdown_object_name = minio_client.generate_object_name(document_version.doc_id, document_version.language,
+        markdown_object_name = minio_client.generate_object_name(tenant_id,
-                                                                 document_version.id, markdown_filename)
+            document_version.doc_id, document_version.language,
+            document_version.id, markdown_filename)
         # Download actual markdown file
         file_data = minio_client.download_document_file(
             tenant_id,