- Improvements to enable deployment in the cloud, mainly changing file access to Minio

- Improvements on RAG logging, and some debugging in that area

common/utils/minio_utils.py

@@ -0,0 +1,86 @@
+from minio import Minio
+from minio.error import S3Error
+from flask import Flask
+import io
+from werkzeug.datastructures import FileStorage
+
+class MinioClient:
+    def __init__(self):
+        self.client = None
+
+    def init_app(self, app: Flask):
+        self.client = Minio(
+            app.config['MINIO_ENDPOINT'],
+            access_key=app.config['MINIO_ACCESS_KEY'],
+            secret_key=app.config['MINIO_SECRET_KEY'],
+            secure=app.config.get('MINIO_USE_HTTPS', False)
+        )
+        app.logger.info(f"MinIO client initialized with endpoint: {app.config['MINIO_ENDPOINT']}")
+
+    def generate_bucket_name(self, tenant_id):
+        return f"tenant-{tenant_id}-bucket"
+
+    def create_tenant_bucket(self, tenant_id):
+        bucket_name = self.generate_bucket_name(tenant_id)
+        try:
+            if not self.client.bucket_exists(bucket_name):
+                self.client.make_bucket(bucket_name)
+                return bucket_name
+            return bucket_name
+        except S3Error as err:
+            raise Exception(f"Error occurred while creating bucket: {err}")
+
+    def generate_object_name(self, document_id, language, version_id, filename):
+        return f"{document_id}/{language}/{version_id}/{filename}"
+
+    def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data):
+        bucket_name = self.generate_bucket_name(tenant_id)
+        object_name = self.generate_object_name(document_id, language, version_id, filename)
+
+        try:
+            if isinstance(file_data, FileStorage):
+                file_data = file_data.read()
+            elif isinstance(file_data, io.BytesIO):
+                file_data = file_data.getvalue()
+            elif isinstance(file_data, str):
+                file_data = file_data.encode('utf-8')
+            elif not isinstance(file_data, bytes):
+                raise TypeError('Unsupported file type. Expected FileStorage, BytesIO, str, or bytes.')
+
+            self.client.put_object(
+                bucket_name, object_name, io.BytesIO(file_data), len(file_data)
+            )
+            return True
+        except S3Error as err:
+            raise Exception(f"Error occurred while uploading file: {err}")
+
+    def download_document_file(self, tenant_id, document_id, language, version_id, filename):
+        bucket_name = self.generate_bucket_name(tenant_id)
+        object_name = self.generate_object_name(document_id, language, version_id, filename)
+        try:
+            response = self.client.get_object(bucket_name, object_name)
+            return response.read()
+        except S3Error as err:
+            raise Exception(f"Error occurred while downloading file: {err}")
+
+    def list_document_files(self, tenant_id, document_id, language=None, version_id=None):
+        bucket_name = self.generate_bucket_name(tenant_id)
+        prefix = f"{document_id}/"
+        if language:
+            prefix += f"{language}/"
+            if version_id:
+                prefix += f"{version_id}/"
+        try:
+            objects = self.client.list_objects(bucket_name, prefix=prefix, recursive=True)
+            return [obj.object_name for obj in objects]
+        except S3Error as err:
+            raise Exception(f"Error occurred while listing files: {err}")
+
+    def delete_document_file(self, tenant_id, document_id, language, version_id, filename):
+        bucket_name = self.generate_bucket_name(tenant_id)
+        object_name = self.generate_object_name(document_id, language, version_id, filename)
+        try:
+            self.client.remove_object(bucket_name, object_name)
+            return True
+        except S3Error as err:
+            raise Exception(f"Error occurred while deleting file: {err}")

common/utils/model_utils.py

@@ -141,7 +141,7 @@ def select_model_variables(tenant):
                                                        default_headers=portkey_headers)
             tool_calling_supported = False
             match llm_model:
-                case 'gpt-4-turbo' | 'gpt-4o':
+                case 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini':
                     tool_calling_supported = True
                 case _:
                     raise Exception(f'Error setting model variables for tenant {tenant.id} '