Cleanup .pyc and .DS_Store, add new modules, remove legacy services

common/utils/asset_utils.py

@@ -5,7 +5,6 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from common.extensions import cache_manager, minio_client, db
 from common.models.interaction import EveAIAsset, EveAIAssetVersion
-from common.utils.document_utils import mark_tenant_storage_dirty
 from common.utils.model_logging_utils import set_logging_information
 
 
@@ -55,7 +54,8 @@ def create_version_for_asset(asset, tenant_id):
 def add_asset_version_file(asset_version, field_name, file, tenant_id):
     object_name, file_size = minio_client.upload_file(asset_version.bucket_name, asset_version.id, field_name,
                                                       file.content_type)
-    mark_tenant_storage_dirty(tenant_id)
+    # mark_tenant_storage_dirty(tenant_id)
+    # TODO - zorg ervoor dat de herberekening van storage onmiddellijk gebeurt!
     return object_name
 
 

common/utils/cache/license_cache.py

@@ -0,0 +1,102 @@
+# common/utils/cache/license_cache.py
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+
+from flask import current_app
+from sqlalchemy import and_
+from sqlalchemy.inspection import inspect
+
+from common.utils.cache.base import CacheHandler
+from common.models.entitlements import License
+
+
+class LicenseCacheHandler(CacheHandler[License]):
+    """Handles caching of active licenses for tenants"""
+    handler_name = 'license_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'active_license')
+        self.configure_keys('tenant_id')
+
+    def _to_cache_data(self, instance: License) -> Dict[str, Any]:
+        """Convert License instance to cache data using SQLAlchemy inspection"""
+        if not instance:
+            return {}
+
+        # Get all column attributes from the SQLAlchemy model
+        mapper = inspect(License)
+        data = {}
+
+        for column in mapper.columns:
+            value = getattr(instance, column.name)
+
+            # Handle date serialization
+            if isinstance(value, dt):
+                data[column.name] = value.isoformat()
+            else:
+                data[column.name] = value
+
+        return data
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> License:
+        """Create License instance from cache data using SQLAlchemy inspection"""
+        if not data:
+            return None
+
+        # Create a new License instance
+        license = License()
+        mapper = inspect(License)
+
+        # Set all attributes dynamically
+        for column in mapper.columns:
+            if column.name in data:
+                value = data[column.name]
+
+                # Handle date deserialization
+                if column.name.endswith('_date') and value:
+                    if isinstance(value, str):
+                        value = dt.fromisoformat(value).date()
+
+                setattr(license, column.name, value)
+
+        return license
+
+    def _should_cache(self, value: License) -> bool:
+        """Validate if the license should be cached"""
+        return value is not None and value.id is not None
+
+    def get_active_license(self, tenant_id: int) -> Optional[License]:
+        """
+        Get the currently active license for a tenant
+
+        Args:
+            tenant_id: ID of the tenant
+
+        Returns:
+            License instance if found, None otherwise
+        """
+
+        def creator_func(tenant_id: int) -> Optional[License]:
+            from common.extensions import db
+            current_date = dt.now(tz=tz.utc).date()
+
+            # TODO --> Active License via active Period?
+
+            return (db.session.query(License)
+                    .filter_by(tenant_id=tenant_id)
+                    .filter(License.start_date <= current_date)
+                    .last())
+
+        return self.get(creator_func, tenant_id=tenant_id)
+
+    def invalidate_tenant_license(self, tenant_id: int):
+        """Invalidate cached license for specific tenant"""
+        self.invalidate(tenant_id=tenant_id)
+
+
+def register_license_cache_handlers(cache_manager) -> None:
+    """Register license cache handlers with cache manager"""
+    cache_manager.register_handler(
+        LicenseCacheHandler,
+        'eveai_model'  # Use existing eveai_model region
+    )

common/utils/document_utils.py

@@ -16,9 +16,15 @@ from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLExce
                                EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
 from ..models.user import Tenant
 from common.utils.model_logging_utils import set_logging_information, update_logging_information
+from common.services.entitlements import LicenseUsageServices
+
+MB_CONVERTOR = 1_048_576
 
 
 def create_document_stack(api_input, file, filename, extension, tenant_id):
+    # Precheck if we can add a document to the stack
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, len(file)/MB_CONVERTOR)
+
     # Create the Document
     catalog_id = int(api_input.get('catalog_id'))
     catalog = Catalog.query.get(catalog_id)
@@ -102,8 +108,6 @@ def create_version_for_document(document, tenant_id, url, sub_file_type,  langua
 
     set_logging_information(new_doc_vers, dt.now(tz.utc))
 
-    mark_tenant_storage_dirty(tenant_id)
-
     return new_doc_vers
 
 
@@ -124,7 +128,7 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
         )
         doc_vers.bucket_name = bn
         doc_vers.object_name = on
-        doc_vers.file_size = size / 1048576  # Convert bytes to MB
+        doc_vers.file_size = size / MB_CONVERTOR  # Convert bytes to MB
 
         db.session.commit()
         current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
@@ -274,6 +278,9 @@ def refresh_document_with_info(doc_id, tenant_id, api_input):
     if not old_doc_vers.url:
         return None, "This document has no URL. Only documents with a URL can be refreshed."
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, old_doc_vers.file_size)
+
     new_doc_vers = create_version_for_document(
         doc, tenant_id,
         old_doc_vers.url,
@@ -330,6 +337,9 @@ def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: byt
 
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, len(file_content) / MB_CONVERTOR)
+
     # Create new version with same file type as original
     extension = old_doc_vers.file_type
 
@@ -377,13 +387,6 @@ def refresh_document(doc_id, tenant_id):
     return refresh_document_with_info(doc_id, tenant_id, api_input)
 
 
-# Function triggered when a document_version is created or updated
-def mark_tenant_storage_dirty(tenant_id):
-    tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first()
-    tenant.storage_dirty = True
-    db.session.commit()
-
-
 def cope_with_local_url(url):
     parsed_url = urlparse(url)
     # Check if this is an internal WordPress URL (TESTING) and rewrite it

common/utils/dynamic_field_utils.py

@@ -0,0 +1,44 @@
+def create_default_config_from_type_config(type_config):
+    """
+    Creëert een dictionary met standaardwaarden gebaseerd op een typedefinitie configuratie.
+
+    Args:
+        type_config (dict): Het configuration-veld van een typedefinitie (bijv. uit processor_types).
+
+    Returns:
+        dict: Een dictionary met de naam van ieder veld als sleutel en de standaardwaarde als waarde.
+        Alleen velden met een standaardwaarde of die verplicht zijn, worden opgenomen.
+
+    Voorbeeld:
+        >>> config = PROCESSOR_TYPES["HTML_PROCESSOR"]["configuration"]
+        >>> create_default_config_from_type_def(config)
+        {'html_tags': 'p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td',
+         'html_end_tags': 'p, li, table',
+         'html_excluded_classes': '',
+         'html_excluded_elements': 'header, footer, nav, script',
+         'html_included_elements': 'article, main',
+         'chunking_heading_level': 2}
+    """
+    if not type_config:
+        return {}
+
+    default_config = {}
+
+    for field_name, field_def in type_config.items():
+        # Als het veld een standaardwaarde heeft, voeg deze toe
+        if "default" in field_def:
+            default_config[field_name] = field_def["default"]
+        # Als het veld verplicht is maar geen standaardwaarde heeft, voeg een lege string toe
+        elif field_def.get("required", False):
+            # Kies een geschikte "lege" waarde op basis van het type
+            field_type = field_def.get("type", "string")
+            if field_type == "string":
+                default_config[field_name] = ""
+            elif field_type == "integer":
+                default_config[field_name] = 0
+            elif field_type == "boolean":
+                default_config[field_name] = False
+            else:
+                default_config[field_name] = ""
+
+    return default_config

common/utils/eveai_exceptions.py

@@ -186,3 +186,65 @@ class EveAINoManagementPartnerForTenant(EveAIException):
         super().__init__(message, status_code, payload)
 
 
+class EveAIQuotaExceeded(EveAIException):
+    """Base exception for quota-related errors"""
+
+    def __init__(self, message, quota_type, current_usage, limit, additional=0, status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+        self.quota_type = quota_type
+        self.current_usage = current_usage
+        self.limit = limit
+        self.additional = additional
+
+
+class EveAIStorageQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when storage quota is exceeded"""
+
+    def __init__(self, current_usage, limit, additional, status_code=400, payload=None):
+        message = (f"Storage quota exceeded. Current: {current_usage:.1f}MB, "
+                   f"Additional: {additional:.1f}MB, Limit: {limit}MB")
+        super().__init__(message, "storage", current_usage, limit, additional, status_code, payload)
+
+
+class EveAIEmbeddingQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when embedding quota is exceeded"""
+
+    def __init__(self, current_usage, limit, additional, status_code=400, payload=None):
+        message = (f"Embedding quota exceeded. Current: {current_usage:.1f}MB, "
+                   f"Additional: {additional:.1f}MB, Limit: {limit}MB")
+        super().__init__(message, "embedding", current_usage, limit, additional, status_code, payload)
+
+
+class EveAIInteractionQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when the interaction token quota is exceeded"""
+
+    def __init__(self, current_usage, limit, status_code=400, payload=None):
+        message = (f"Interaction token quota exceeded. Current: {current_usage:.2f}M tokens, "
+                   f"Limit: {limit:.2f}M tokens")
+        super().__init__(message, "interaction", current_usage, limit, 0, status_code, payload)
+
+
+class EveAIQuotaWarning(EveAIException):
+    """Warning when approaching quota limits (not blocking)"""
+
+    def __init__(self, message, quota_type, usage_percentage, status_code=200, payload=None):
+        super().__init__(message, status_code, payload)
+        self.quota_type = quota_type
+        self.usage_percentage = usage_percentage
+
+
+class EveAILicensePeriodsExceeded(EveAIException):
+    """Raised when no more license periods can be created for a given license"""
+
+    def __init__(self, license_id, status_code=400, payload=None):
+        message = f"No more license periods can be created for license with ID {license_id}. "
+        super().__init__(message, status_code, payload)
+
+
+class EveAIPendingLicensePeriod(EveAIException):
+    """Raised when a license period is pending"""
+
+    def __init__(self, status_code=400, payload=None):
+        message = f"Basic Fee Payment has not been received yet. Please ensure payment has been made, and please wait for payment to be processed."
+        super().__init__(message, status_code, payload)
+

common/utils/mail_utils.py

@@ -0,0 +1,46 @@
+from scaleway import Client
+from scaleway.tem.v1alpha1.api import TemV1Alpha1API
+from scaleway.tem.v1alpha1.types import CreateEmailRequestAddress
+from html2text import HTML2Text
+from flask import current_app
+
+
+def send_email(to_email, to_name, subject, html):
+    current_app.logger.debug(f"Sending email to {to_email} with subject {subject}")
+    access_key = current_app.config['SW_EMAIL_ACCESS_KEY']
+    secret_key = current_app.config['SW_EMAIL_SECRET_KEY']
+    default_project_id = current_app.config['SW_PROJECT']
+    default_region = "fr-par"
+    current_app.logger.debug(f"Access Key: {access_key}\nSecret Key: {secret_key}\n"
+                             f"Default Project ID: {default_project_id}\nDefault Region: {default_region}")
+    client = Client(
+        access_key=access_key,
+        secret_key=secret_key,
+        default_project_id=default_project_id,
+        default_region=default_region
+    )
+    current_app.logger.debug(f"Scaleway Client Initialized")
+    tem = TemV1Alpha1API(client)
+    current_app.logger.debug(f"Tem Initialized")
+    from_ = CreateEmailRequestAddress(email=current_app.config['SW_EMAIL_SENDER'],
+                                      name=current_app.config['SW_EMAIL_NAME'])
+    to_ = CreateEmailRequestAddress(email=to_email, name=to_name)
+
+    email = tem.create_email(
+        from_=from_,
+        to=[to_],
+        subject=subject,
+        text=html_to_text(html),
+        html=html,
+        project_id=default_project_id,
+    )
+    current_app.logger.debug(f"Email sent to {to_email}")
+
+
+def html_to_text(html_content):
+    """Convert HTML to plain text using html2text"""
+    h = HTML2Text()
+    h.ignore_images = True
+    h.ignore_emphasis = False
+    h.body_width = 0  # No wrapping
+    return h.handle(html_content)

common/utils/middleware.py

@@ -4,11 +4,11 @@ for handling tenant requests
 """
 
 from flask_security import current_user
-from flask import session, current_app, redirect
+from flask import session
 from .database import Database
 from .eveai_exceptions import EveAINoSessionTenant, EveAINoSessionPartner, EveAINoManagementPartnerService, \
     EveAINoManagementPartnerForTenant
-from ..services.user_services import UserServices
+from common.services.user import UserServices
 
 
 def mw_before_request():

common/utils/model_logging_utils.py

@@ -10,7 +10,6 @@ def set_logging_information(obj, timestamp):
         obj.created_by = user_id
         obj.updated_by = user_id
 
-
 def update_logging_information(obj, timestamp):
     obj.updated_at = timestamp
 

common/utils/security.py

@@ -39,11 +39,12 @@ def is_valid_tenant(tenant_id):
         raise EveAITenantInvalid(tenant_id)
     else:
         current_date = dt.now(tz=tz.utc).date()
-        active_license = (License.query.filter_by(tenant_id=tenant_id)
-                          .filter(and_(License.start_date <= current_date,
-                                       License.end_date >= current_date))
-                          .one_or_none())
-        if not active_license:
-            raise EveAINoActiveLicense(tenant_id)
+        # TODO -> Check vervangen door Active License Period!
+        # active_license = (License.query.filter_by(tenant_id=tenant_id)
+        #                   .filter(and_(License.start_date <= current_date,
+        #                                License.end_date >= current_date))
+        #                   .one_or_none())
+        # if not active_license:
+        #     raise EveAINoActiveLicense(tenant_id)
 
     return True

common/utils/security_utils.py

@@ -1,11 +1,10 @@
 from flask import current_app, render_template
 from flask_security import current_user
-from flask_mailman import EmailMessage
 from itsdangerous import URLSafeTimedSerializer
-import socket
 
 from common.models.user import Role
 from common.utils.nginx_utils import prefixed_url_for
+from common.utils.mail_utils import send_email
 
 
 def confirm_token(token, expiration=3600):
@@ -18,14 +17,6 @@ def confirm_token(token, expiration=3600):
     return email
 
 
-def send_email(to, subject, template):
-    msg = EmailMessage(subject=subject,
-                       body=template,
-                       to=[to])
-    msg.content_subtype = "html"
-    msg.send()
-
-
 def generate_reset_token(email):
     serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY'])
     return serializer.dumps(email, salt=current_app.config['SECURITY_PASSWORD_SALT'])
@@ -37,9 +28,6 @@ def generate_confirmation_token(email):
 
 
 def send_confirmation_email(user):
-    if not test_smtp_connection():
-        raise Exception("Failed to connect to SMTP server")
-
     token = generate_confirmation_token(user.email)
     confirm_url = prefixed_url_for('security_bp.confirm_email', token=token, _external=True)
 
@@ -47,7 +35,7 @@ def send_confirmation_email(user):
     subject = "Please confirm your email"
 
     try:
-        send_email(user.email, "Confirm your email", html)
+        send_email(user.email, f"{user.first_name} {user.last_name}", "Confirm your email", html)
         current_app.logger.info(f'Confirmation email sent to {user.email}')
     except Exception as e:
         current_app.logger.error(f'Failed to send confirmation email to {user.email}. Error: {str(e)}')
@@ -62,41 +50,13 @@ def send_reset_email(user):
     subject = "Reset Your Password"
 
     try:
-        send_email(user.email, "Reset Your Password", html)
+        send_email(user.email, f"{user.first_name} {user.last_name}", subject, html)
         current_app.logger.info(f'Reset email sent to {user.email}')
     except Exception as e:
         current_app.logger.error(f'Failed to send reset email to {user.email}. Error: {str(e)}')
         raise
 
 
-def test_smtp_connection():
-    try:
-        current_app.logger.info(f"Attempting to resolve google.com...")
-        google_ip = socket.gethostbyname('google.com')
-        current_app.logger.info(f"Successfully resolved google.com to {google_ip}")
-    except Exception as e:
-        current_app.logger.error(f"Failed to resolve google.com: {str(e)}")
-
-    try:
-        smtp_server = current_app.config['MAIL_SERVER']
-        current_app.logger.info(f"Attempting to resolve {smtp_server}...")
-        smtp_ip = socket.gethostbyname(smtp_server)
-        current_app.logger.info(f"Successfully resolved {smtp_server} to {smtp_ip}")
-    except Exception as e:
-        current_app.logger.error(f"Failed to resolve {smtp_server}: {str(e)}")
-
-    try:
-        smtp_server = current_app.config['MAIL_SERVER']
-        smtp_port = current_app.config['MAIL_PORT']
-        sock = socket.create_connection((smtp_server, smtp_port), timeout=10)
-        sock.close()
-        current_app.logger.info(f"Successfully connected to SMTP server {smtp_server}:{smtp_port}")
-        return True
-    except Exception as e:
-        current_app.logger.error(f"Failed to connect to SMTP server: {str(e)}")
-        return False
-
-
 def get_current_user_roles():
     """Get the roles of the currently authenticated user.
 

common/utils/template_filters.py

@@ -29,9 +29,23 @@ def time_difference(start_dt, end_dt):
     return "Ongoing"
 
 
+def status_color(status_name):
+    """Return Bootstrap color class for status"""
+    colors = {
+        'UPCOMING': 'secondary',
+        'PENDING': 'warning',
+        'ACTIVE': 'success',
+        'COMPLETED': 'info',
+        'INVOICED': 'primary',
+        'CLOSED': 'dark'
+    }
+    return colors.get(status_name, 'secondary')
+
+
 def register_filters(app):
     """
     Registers custom filters with the Flask app.
     """
     app.jinja_env.filters['to_local_time'] = to_local_time
     app.jinja_env.filters['time_difference'] = time_difference
+    app.jinja_env.filters['status_color'] = status_color