Cleanup .pyc and .DS_Store, add new modules, remove legacy services

common/utils/document_utils.py

@@ -16,9 +16,15 @@ from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLExce
                                EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
 from ..models.user import Tenant
 from common.utils.model_logging_utils import set_logging_information, update_logging_information
+from common.services.entitlements import LicenseUsageServices
+
+MB_CONVERTOR = 1_048_576
 
 
 def create_document_stack(api_input, file, filename, extension, tenant_id):
+    # Precheck if we can add a document to the stack
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, len(file)/MB_CONVERTOR)
+
     # Create the Document
     catalog_id = int(api_input.get('catalog_id'))
     catalog = Catalog.query.get(catalog_id)
@@ -102,8 +108,6 @@ def create_version_for_document(document, tenant_id, url, sub_file_type,  langua
 
     set_logging_information(new_doc_vers, dt.now(tz.utc))
 
-    mark_tenant_storage_dirty(tenant_id)
-
     return new_doc_vers
 
 
@@ -124,7 +128,7 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
         )
         doc_vers.bucket_name = bn
         doc_vers.object_name = on
-        doc_vers.file_size = size / 1048576  # Convert bytes to MB
+        doc_vers.file_size = size / MB_CONVERTOR  # Convert bytes to MB
 
         db.session.commit()
         current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
@@ -274,6 +278,9 @@ def refresh_document_with_info(doc_id, tenant_id, api_input):
     if not old_doc_vers.url:
         return None, "This document has no URL. Only documents with a URL can be refreshed."
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, old_doc_vers.file_size)
+
     new_doc_vers = create_version_for_document(
         doc, tenant_id,
         old_doc_vers.url,
@@ -330,6 +337,9 @@ def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: byt
 
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, len(file_content) / MB_CONVERTOR)
+
     # Create new version with same file type as original
     extension = old_doc_vers.file_type
 
@@ -377,13 +387,6 @@ def refresh_document(doc_id, tenant_id):
     return refresh_document_with_info(doc_id, tenant_id, api_input)
 
 
-# Function triggered when a document_version is created or updated
-def mark_tenant_storage_dirty(tenant_id):
-    tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first()
-    tenant.storage_dirty = True
-    db.session.commit()
-
-
 def cope_with_local_url(url):
     parsed_url = urlparse(url)
     # Check if this is an internal WordPress URL (TESTING) and rewrite it