- Adding functionality for listing and editing assets

- Started adding functionality for creating a 'full_documents' list view.

common/utils/document_utils.py

@@ -15,12 +15,11 @@ from config.type_defs.processor_types import PROCESSOR_TYPES
 from .config_field_types import normalize_json_field
 from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
                                EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
+from .minio_utils import MIB_CONVERTOR
 from ..models.user import Tenant
 from common.utils.model_logging_utils import set_logging_information, update_logging_information
 from common.services.entitlements import LicenseUsageServices
 
-MB_CONVERTOR = 1_048_576
-
 
 def get_file_size(file):
     try:
@@ -39,7 +38,7 @@ def get_file_size(file):
 def create_document_stack(api_input, file, filename, extension, tenant_id):
     # Precheck if we can add a document to the stack
 
-    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file)/MB_CONVERTOR)
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file) / MIB_CONVERTOR)
 
     # Create the Document
     catalog_id = int(api_input.get('catalog_id'))
@@ -144,7 +143,7 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
         )
         doc_vers.bucket_name = bn
         doc_vers.object_name = on
-        doc_vers.file_size = size / MB_CONVERTOR  # Convert bytes to MB
+        doc_vers.file_size = size / MIB_CONVERTOR  # Convert bytes to MB
 
         db.session.commit()
         current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
@@ -354,7 +353,7 @@ def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: byt
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
     # Precheck if we have enough quota for the new version
-    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file_content) / MB_CONVERTOR)
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file_content) / MIB_CONVERTOR)
 
     # Create new version with same file type as original
     extension = old_doc_vers.file_type

common/utils/filtered_list_view.py

@@ -1,54 +0,0 @@
-from flask import request, render_template, abort
-from sqlalchemy import desc, asc
-
-
-class FilteredListView:
-    def __init__(self, model, template, per_page=10):
-        self.model = model
-        self.template = template
-        self.per_page = per_page
-
-    def get_query(self):
-        return self.model.query
-
-    def apply_filters(self, query):
-        filters = request.args.get('filters', {})
-        for key, value in filters.items():
-            if hasattr(self.model, key):
-                column = getattr(self.model, key)
-                if value.startswith('like:'):
-                    query = query.filter(column.like(f"%{value[5:]}%"))
-                else:
-                    query = query.filter(column == value)
-        return query
-
-    def apply_sorting(self, query):
-        sort_by = request.args.get('sort_by')
-        if sort_by and hasattr(self.model, sort_by):
-            sort_order = request.args.get('sort_order', 'asc')
-            column = getattr(self.model, sort_by)
-            if sort_order == 'desc':
-                query = query.order_by(desc(column))
-            else:
-                query = query.order_by(asc(column))
-        return query
-
-    def paginate(self, query):
-        page = request.args.get('page', 1, type=int)
-        return query.paginate(page=page, per_page=self.per_page, error_out=False)
-
-    def get(self):
-        query = self.get_query()
-        query = self.apply_filters(query)
-        query = self.apply_sorting(query)
-        pagination = self.paginate(query)
-
-        context = {
-            'items': pagination.items,
-            'pagination': pagination,
-            'model': self.model.__name__,
-            'filters': request.args.get('filters', {}),
-            'sort_by': request.args.get('sort_by'),
-            'sort_order': request.args.get('sort_order', 'asc')
-        }
-        return render_template(self.template, **context)

common/utils/minio_utils.py

@@ -4,6 +4,9 @@ from flask import Flask
 import io
 from werkzeug.datastructures import FileStorage
 
+MIB_CONVERTOR = 1_048_576
+
+
 class MinioClient:
     def __init__(self):
         self.client = None
@@ -58,7 +61,7 @@ class MinioClient:
             raise Exception(f"Error occurred while uploading file: {err}")
 
     def upload_asset_file(self, tenant_id: int, asset_id: int, asset_type: str, file_type: str,
-                          file_data: bytes | FileStorage | io.BytesIO | str,) -> tuple[str, str, int]:
+                          file_data: bytes | FileStorage | io.BytesIO | str, ) -> tuple[str, str, int]:
         bucket_name = self.generate_bucket_name(tenant_id)
         object_name = self.generate_asset_name(asset_id, asset_type, file_type)
 
@@ -119,4 +122,11 @@ class MinioClient:
         try:
             self.client.remove_object(bucket_name, object_name)
         except S3Error as err:
-            raise Exception(f"Error occurred while deleting object: {err}")
+            raise Exception(f"Error occurred while deleting object: {err}")
+
+    def get_bucket_size(self, tenant_id: int) -> int:
+        bucket_name = self.generate_bucket_name(tenant_id)
+        total_size = 0
+        for obj in self.client.list_objects(bucket_name, recursive=True):
+            total_size += obj.size
+        return total_size