- Fixed bug where negative answers in KO Criteria resulted in a blank answer

- Fixed bug where removal of audio processor caused eveai_workers to not start up, resulting in documents not being processed.

.aiignore

@@ -0,0 +1,19 @@
+# An .aiignore file follows the same syntax as a .gitignore file.
+# .gitignore documentation: https://git-scm.com/docs/gitignore
+
+# you can ignore files
+.DS_Store
+*.log
+*.tmp
+
+# or folders
+dist/
+build/
+out/
+nginx/node_modules/
+nginx/static/
+db_backups/
+docker/eveai_logs/
+docker/logs/
+docker/minio/
+

.gitignore

@@ -14,7 +14,6 @@ __pycache__
 **/__pycache__
 /.idea
 *.pyc
-*.pyc
 common/.DS_Store
 common/__pycache__/__init__.cpython-312.pyc
 common/__pycache__/extensions.cpython-312.pyc
@@ -52,3 +51,7 @@ scripts/__pycache__/run_eveai_app.cpython-312.pyc
 /patched_packages/crewai/
 /docker/prometheus/data/
 /docker/grafana/data/
+/temp_requirements/
+/nginx/node_modules/
+/nginx/.parcel-cache/
+/nginx/static/

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/eveAI.iml

@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="Flask">
-    <option name="enabled" value="true" />
-  </component>
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/.venv" />
-      <excludeFolder url="file://$MODULE_DIR$/.venv2" />
-    </content>
-    <orderEntry type="jdk" jdkName="Python 3.12 (eveai_dev)" jdkType="Python SDK" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="TemplatesService">
-    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
-    <option name="TEMPLATE_FOLDERS">
-      <list>
-        <option value="$MODULE_DIR$/templates" />
-      </list>
-    </option>
-  </component>
-</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Black">
-    <option name="sdkName" value="Python 3.12 (eveai_tbd)" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (TBD)" project-jdk-type="Python SDK" />
-</project>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/TBD.iml" filepath="$PROJECT_DIR$/.idea/TBD.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

CHANGELOG.md

@@ -5,19 +5,79 @@ All notable changes to EveAI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [2.3.0-alfa]
 
 ### Added
-- For new features.
+- Introduction of Push Gateway for Prometheus
+- Introduction of Partner Models
+- Introduction of Tenant and Partner codes for more security
+- Introduction of 'Management Partner' type and additional 'Partner Admin'-role
+- Introduction of a technical services layer
+- Introduction of partner-specific configurations
+- Introduction of additional test environment
+- Introduction of strict no-overage usage
+- Introduction of LicensePeriod, Payments & Invoices
+- Introduction of Processed File Viewer
+- Introduction of Traicie Role Definition Specialist
+- Allow invocation of non-interactive specialists in administrative interface (eveai_app)
+- Introduction of advanced JSON editor
+- Introduction of ChatSession (Specialist Execution) follow-up in administrative interface
+- Introduce npm for javascript libraries usage and optimisations
+- Introduction of new top bar in administrative interface to show session defaults (removing old navbar buttons)
+- 
 
 ### Changed
-- For changes in existing functionality.
+- Add 'Register'-button to list views, replacing register menu-items
+- Add additional environment capabilities in docker
+- PDF Processor now uses Mistral OCR
+- Allow additional chunking mechanisms for very long chunks (in case of very large documents)
+- Allow for TrackedMistralAIEmbedding batching to allow for processing long documents
+- RAG & SPIN Specialist improvements
+- Move mail messaging from standard SMTP to Scaleway TEM mails
+- Improve mail layouts
+- Add functionality to add a default dictionary for dynamic forms
+- AI model choices defined by Ask Eve AI iso Tenant (replaces ModelVariables completely)
+- Improve HTML Processing
+- Pagination improvements
+- Update Material Kit Pro to latest version
+
+### Removed
+- Repopack implementation ==> Using PyCharm's new AI capabilities instead
+
+### Fixed
+- Synchronous vs Asynchronous behaviour in crewAI type specialists
+- Nasty dynamic boolean fields bug corrected
+- Several smaller bugfixes
+- Tasks & Tools editors finished
+
+### Security
+- In case of vulnerabilities.
+
+## [2.2.0-alfa]
+
+### Added
+- Mistral AI as main provider for embeddings, chains and specialists
+- Usage measuring for specialists
+- RAG from chain to specialist technology
+- Dossier catalog management possibilities added to eveai_app
+- Asset definition (Paused - other priorities)
+- Prometheus and Grafana
+- Add prometheus monitoring to business events
+- Asynchronous execution of specialists
+
+### Changed
+- Moved choice for AI providers / models to specialists and prompts
+- Improve RAG to not repeat historic answers
+- Fixed embedding model, no more choices allowed
+- clean url (of tracking parameters) before adding it to a catalog
 
 ### Deprecated
 - For soon-to-be removed features.
 
 ### Removed
-- For now removed features.
+- Add Multiple URLs removed from menu
+- Old Specialist items removed from interaction menu
+- 
 
 ### Fixed
 - Set default language when registering Documents or URLs.

Evie Overview.md

@@ -0,0 +1,85 @@
+# Evie Overview
+
+Owner: pieter Laroy
+
+# Introduction
+
+The Evie project (developed by AskEveAI) is a SAAS product that enables SMEs to easily introduce AI optimisations for both internal and external use. There are two big concepts:
+
+- Catalogs: these allow tenants to store information about their organisations or enterprises
+- Specialists: these allow tenants to perform logic into their processes, communications, …
+
+As such, we could say we have an advanced RAG system tenants can use to optimise their workings.
+
+## Multi-tenant
+
+The application has a multi-tenant setup built in. This is reflected in:
+
+- The Database:
+    - We have 1 public schema, in which general information is defined such as tenants, their users, domains, licenses, …
+    - We have a schema (named 1, 2, …) for each of the tenants defined in the system, containing all information on the tenant’s catalogs & documents, specialists & interactions, …
+- File Storage
+    - We use S3-compatible storage
+    - A bucket is defined for each tenant, storing their specific documents, assets, …
+
+That way, general information required for the operation of Evie is stored in the public schema, and specific and potentially sensitive information is nicely stored behind a Chinese wall for each of the tenants.
+
+## Partners
+
+We started to define the concept of a partner. This allows us to have partners that introduce tenants to Evie, or offer them additional functionality (specialists) or knowledge (catalogs). This concept is in an early stage at this point.
+
+## Domains
+
+In order to ensure a structured approach, we have defined several domains in the project:
+
+- **User**: the user domain is used to store all data on partners, tenants, actual users.
+- **Document**: the document domain is used to store all information on catalogs, documents, how to process documents, …
+- **Interaction**: This domain allows us to define specialists, agents, … and to interact with the specialists and agents.
+- **Entitlements**: This domain defines all license information, usage, …
+
+# Project Structure
+
+## Common
+
+The common folder contains code that is used in different components of the system. It contains the following important pieces:
+
+- **models**: in the models folder you can find the SQLAlchemy models used throughout the application. These models are organised in their relevant domains.
+- **eveai_model**: some classes to handle usage, wrappers around standard LLM clients
+- **langchain**: similar to eveai_model, but in the langchain library
+- **services**: I started to define services to define reusable functionality in the system. There again are defined in their respective domains
+- **utils**: a whole bunch of utility classes. Some should get converted to services classes in the future
+- **utils/cache**: contains code for caching different elements in the application
+
+## config
+
+The config folder contains quite some configuration data (as the name suggests):
+
+- **config.py**: general configuration
+- **logging_config.py**: definition of logging files
+- **model_config.py**: obsolete
+- **type_defs**: contains the lists of definitions for several types used throughout the application. E.g. processor_types, specialist_types, …
+- **All other folders**: detailed configuration of all the types defined in type_defs.
+
+## docker
+
+The docker folder contains the configuration and scripts used for all operations on configuring and building containers, distributing containers, …
+
+## eveai_… folders
+
+These are different components (containerized) of our application:
+
+- **eveai_api**: The API of our application.
+- **eveai_app**: The administrative interface of our application.
+- **eveai_beat**: a means to install batch processes for our application.
+- **eveai_chat**: obsolete at this moment
+- **eveai_chat_workers**: celery based invocation of our specialists
+- **eveai_client**: newly added. A desktop client to invoking specialists.
+- **eveai_entitlements**: celery based approach to handling business events, measuring and updating usage, …
+- **eveai_workers**: celery based approach to filling catalogs with documents (embedding)
+
+## Remaining folders
+
+- **integrations**: integrations to e.g. Wordpress and Zapier.
+- **migrations**: SQLAlchemy database migration files (for public and tenant schema)
+- **nginx**: configuration and static files for nginx
+- **scripts**: various scripts used to start up components, to perform database operations, …

common/eveai_model/tracked_mistral_embeddings.py

@@ -9,32 +9,128 @@ from mistralai import Mistral
 
 
 class TrackedMistralAIEmbeddings(EveAIEmbeddings):
-    def __init__(self, model: str = "mistral_embed"):
+    def __init__(self, model: str = "mistral_embed", batch_size: int = 10):
+        """
+        Initialize the TrackedMistralAIEmbeddings class.
+
+        Args:
+            model: The embedding model to use
+            batch_size: Maximum number of texts to send in a single API call
+        """
         api_key = current_app.config['MISTRAL_API_KEY']
         self.client = Mistral(
             api_key=api_key
         )
         self.model = model
+        self.batch_size = batch_size
         super().__init__()
 
     def embed_documents(self, texts: list[str]) -> list[list[float]]:
-        start_time = time.time()
-        result = self.client.embeddings.create(
-            model=self.model,
-            inputs=texts
-        )
-        end_time = time.time()
+        """
+        Embed a list of texts, processing in batches to avoid API limitations.
 
-        metrics = {
-            'total_tokens': result.usage.total_tokens,
-            'prompt_tokens': result.usage.prompt_tokens,  # For embeddings, all tokens are prompt tokens
-            'completion_tokens': result.usage.completion_tokens,
-            'time_elapsed': end_time - start_time,
-            'interaction_type': 'Embedding',
-        }
-        current_event.log_llm_metrics(metrics)
+        Args:
+            texts: A list of texts to embed
 
-        embeddings = [embedding.embedding for embedding in result.data]
+        Returns:
+            A list of embeddings, one for each input text
+        """
+        if not texts:
+            return []
 
-        return embeddings
+        all_embeddings = []
+
+        # Process texts in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i:i + self.batch_size]
+            batch_num = i // self.batch_size + 1
+
+            start_time = time.time()
+            try:
+                result = self.client.embeddings.create(
+                    model=self.model,
+                    inputs=batch
+                )
+                end_time = time.time()
+                batch_time = end_time - start_time
+
+                batch_embeddings = [embedding.embedding for embedding in result.data]
+                all_embeddings.extend(batch_embeddings)
+
+                # Log metrics for this batch
+                metrics = {
+                    'total_tokens': result.usage.total_tokens,
+                    'prompt_tokens': result.usage.prompt_tokens,
+                    'completion_tokens': result.usage.completion_tokens,
+                    'time_elapsed': batch_time,
+                    'interaction_type': 'Embedding',
+                    'batch': batch_num,
+                    'batch_size': len(batch)
+                }
+                current_event.log_llm_metrics(metrics)
+
+                # If processing multiple batches, add a small delay to avoid rate limits
+                if len(texts) > self.batch_size and i + self.batch_size < len(texts):
+                    time.sleep(0.25)  # 250ms pause between batches
+
+            except Exception as e:
+                current_app.logger.error(f"Error in embedding batch {batch_num}: {str(e)}")
+                # If a batch fails, try to process each text individually
+                for j, text in enumerate(batch):
+                    try:
+                        single_start_time = time.time()
+                        single_result = self.client.embeddings.create(
+                            model=self.model,
+                            inputs=[text]
+                        )
+                        single_end_time = time.time()
+
+                        # Add the single embedding
+                        single_embedding = single_result.data[0].embedding
+                        all_embeddings.append(single_embedding)
+
+                        # Log metrics for this individual embedding
+                        single_metrics = {
+                            'total_tokens': single_result.usage.total_tokens,
+                            'prompt_tokens': single_result.usage.prompt_tokens,
+                            'completion_tokens': single_result.usage.completion_tokens,
+                            'time_elapsed': single_end_time - single_start_time,
+                            'interaction_type': 'Embedding',
+                            'batch': f"{batch_num}-recovery-{j}",
+                            'batch_size': 1
+                        }
+                        current_event.log_llm_metrics(single_metrics)
+
+                    except Exception as inner_e:
+                        current_app.logger.error(f"Failed to embed individual text at index {i + j}: {str(inner_e)}")
+                        # Add a zero vector as a placeholder for failed embeddings
+                        # Use the correct dimensionality for the model (1024 for mistral_embed)
+                        embedding_dim = 1024
+                        all_embeddings.append([0.0] * embedding_dim)
+
+        total_batches = (len(texts) + self.batch_size - 1) // self.batch_size
+        current_app.logger.info(f"Embedded {len(texts)} texts in {total_batches} batches")
+
+        return all_embeddings
+
+    # def embed_documents(self, texts: list[str]) -> list[list[float]]:
+    #     start_time = time.time()
+    #     result = self.client.embeddings.create(
+    #         model=self.model,
+    #         inputs=texts
+    #     )
+    #     end_time = time.time()
+    #
+    #     metrics = {
+    #         'total_tokens': result.usage.total_tokens,
+    #         'prompt_tokens': result.usage.prompt_tokens,  # For embeddings, all tokens are prompt tokens
+    #         'completion_tokens': result.usage.completion_tokens,
+    #         'time_elapsed': end_time - start_time,
+    #         'interaction_type': 'Embedding',
+    #     }
+    #     current_event.log_llm_metrics(metrics)
+    #
+    #     embeddings = [embedding.embedding for embedding in result.data]
+    #
+    #     return embeddings
 

common/eveai_model/tracked_mistral_ocr_client.py

@@ -0,0 +1,53 @@
+import re
+import time
+
+from flask import current_app
+from mistralai import Mistral
+
+from common.utils.business_event_context import current_event
+
+
+class TrackedMistralOcrClient:
+    def __init__(self):
+        api_key = current_app.config['MISTRAL_API_KEY']
+        self.client = Mistral(
+            api_key=api_key,
+        )
+        self.model = "mistral-ocr-latest"
+
+    def _get_title(self, markdown):
+        # Look for the first level-1 heading
+        match = re.search(r'^# (.+)', markdown, re.MULTILINE)
+        return match.group(1).strip() if match else None
+
+    def process_pdf(self, file_name, file_content):
+        start_time = time.time()
+        uploaded_pdf = self.client.files.upload(
+            file={
+                "file_name": file_name,
+                "content": file_content
+            },
+            purpose="ocr"
+        )
+        signed_url = self.client.files.get_signed_url(file_id=uploaded_pdf.id)
+        ocr_response = self.client.ocr.process(
+            model=self.model,
+            document={
+                "type": "document_url",
+                "document_url": signed_url.url
+            },
+            include_image_base64=False
+        )
+        nr_of_pages = len(ocr_response.pages)
+        all_markdown = " ".join(page.markdown for page in ocr_response.pages)
+        title = self._get_title(all_markdown)
+        end_time = time.time()
+
+        metrics = {
+            'nr_of_pages': nr_of_pages,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'OCR',
+        }
+        current_event.log_llm_metrics(metrics)
+
+        return all_markdown, title

common/extensions.py

@@ -2,7 +2,6 @@ from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
 from flask_bootstrap import Bootstrap
 from flask_security import Security
-from flask_mailman import Mail
 from flask_login import LoginManager
 from flask_cors import CORS
 from flask_jwt_extended import JWTManager
@@ -11,11 +10,10 @@ from flask_wtf import CSRFProtect
 from flask_restx import Api
 from prometheus_flask_exporter import PrometheusMetrics
 
-from .langchain.templates.template_manager import TemplateManager
 from .utils.cache.eveai_cache_manager import EveAICacheManager
+from .utils.content_utils import ContentManager
 from .utils.simple_encryption import SimpleEncryption
 from .utils.minio_utils import MinioClient
-from .utils.performance_monitoring import EveAIMetrics
 
 
 # Create extensions
@@ -24,7 +22,6 @@ migrate = Migrate()
 bootstrap = Bootstrap()
 csrf = CSRFProtect()
 security = Security()
-mail = Mail()
 login_manager = LoginManager()
 cors = CORS()
 jwt = JWTManager()
@@ -33,7 +30,6 @@ api_rest = Api()
 simple_encryption = SimpleEncryption()
 minio_client = MinioClient()
 metrics = PrometheusMetrics.for_app_factory()
-template_manager = TemplateManager()
 cache_manager = EveAICacheManager()
-eveai_metrics = EveAIMetrics()
+content_manager = ContentManager()
 

common/langchain/llm_metrics_handler.py

@@ -3,7 +3,6 @@ from langchain.callbacks.base import BaseCallbackHandler
 from typing import Dict, Any, List
 from langchain.schema import LLMResult
 from common.utils.business_event_context import current_event
-from flask import current_app
 
 
 class LLMMetricsHandler(BaseCallbackHandler):

common/langchain/persistent_llm_metrics_handler.py

@@ -0,0 +1,47 @@
+import time
+from langchain.callbacks.base import BaseCallbackHandler
+from typing import Dict, Any, List
+from langchain.schema import LLMResult
+from common.utils.business_event_context import current_event
+
+
+class PersistentLLMMetricsHandler(BaseCallbackHandler):
+    """Metrics handler that allows metrics to be retrieved from within any call. In case metrics are required for other
+    purposes than business event logging."""
+
+    def __init__(self):
+        self.total_tokens: int = 0
+        self.prompt_tokens: int = 0
+        self.completion_tokens: int = 0
+        self.start_time: float = 0
+        self.end_time: float = 0
+        self.total_time: float = 0
+
+    def reset(self):
+        self.total_tokens = 0
+        self.prompt_tokens = 0
+        self.completion_tokens = 0
+        self.start_time = 0
+        self.end_time = 0
+        self.total_time = 0
+
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        self.start_time = time.time()
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        self.end_time = time.time()
+        self.total_time = self.end_time - self.start_time
+
+        usage = response.llm_output.get('token_usage', {})
+        self.prompt_tokens += usage.get('prompt_tokens', 0)
+        self.completion_tokens += usage.get('completion_tokens', 0)
+        self.total_tokens = self.prompt_tokens + self.completion_tokens
+
+    def get_metrics(self) -> Dict[str, int | float]:
+        return {
+            'total_tokens': self.total_tokens,
+            'prompt_tokens': self.prompt_tokens,
+            'completion_tokens': self.completion_tokens,
+            'time_elapsed': self.total_time,
+            'interaction_type': 'LLM',
+        }

common/langchain/templates/template_manager.py

@@ -1,153 +0,0 @@
-import os
-import yaml
-from typing import Dict, Optional, Any
-from packaging import version
-from dataclasses import dataclass
-from flask import current_app, Flask
-
-from common.utils.os_utils import get_project_root
-
-
-@dataclass
-class PromptTemplate:
-    """Represents a versioned prompt template"""
-    content: str
-    version: str
-    metadata: Dict[str, Any]
-
-
-class TemplateManager:
-    """Manages versioned prompt templates"""
-
-    def __init__(self):
-        self.templates_dir = None
-        self._templates = None
-        self.app = None
-
-    def init_app(self, app: Flask) -> None:
-        # Initialize template manager
-        base_dir = "/app"
-        self.templates_dir = os.path.join(base_dir, 'config', 'prompts')
-        self.app = app
-        self._templates = self._load_templates()
-        # Log available templates for each supported model
-        for llm in app.config['SUPPORTED_LLMS']:
-            try:
-                available_templates = self.list_templates(llm)
-                app.logger.info(f"Loaded templates for {llm}: {available_templates}")
-            except ValueError:
-                app.logger.warning(f"No templates found for {llm}")
-
-    def _load_templates(self) -> Dict[str, Dict[str, Dict[str, PromptTemplate]]]:
-        """
-        Load all template versions from the templates directory.
-        Structure: {provider.model -> {template_name -> {version -> template}}}
-        Directory structure:
-        prompts/
-          ├── provider/
-          │   └── model/
-          │       └── template_name/
-          │           └── version.yaml
-        """
-        templates = {}
-
-        # Iterate through providers (anthropic, openai)
-        for provider in os.listdir(self.templates_dir):
-            provider_path = os.path.join(self.templates_dir, provider)
-            if not os.path.isdir(provider_path):
-                continue
-
-            # Iterate through models (claude-3, gpt-4o)
-            for model in os.listdir(provider_path):
-                model_path = os.path.join(provider_path, model)
-                if not os.path.isdir(model_path):
-                    continue
-
-                provider_model = f"{provider}.{model}"
-                templates[provider_model] = {}
-
-                # Iterate through template types (rag, summary, etc.)
-                for template_name in os.listdir(model_path):
-                    template_path = os.path.join(model_path, template_name)
-                    if not os.path.isdir(template_path):
-                        continue
-
-                    template_versions = {}
-                    # Load all version files for this template
-                    for version_file in os.listdir(template_path):
-                        if not version_file.endswith('.yaml'):
-                            continue
-
-                        version_str = version_file[:-5]  # Remove .yaml
-                        if not self._is_valid_version(version_str):
-                            current_app.logger.warning(
-                                f"Invalid version format for {template_name}: {version_str}")
-                            continue
-
-                        try:
-                            with open(os.path.join(template_path, version_file)) as f:
-                                template_data = yaml.safe_load(f)
-                                # Verify required fields
-                                if not template_data.get('content'):
-                                    raise ValueError("Template content is required")
-
-                                template_versions[version_str] = PromptTemplate(
-                                    content=template_data['content'],
-                                    version=version_str,
-                                    metadata=template_data.get('metadata', {})
-                                )
-                        except Exception as e:
-                            current_app.logger.error(
-                                f"Error loading template {template_name} version {version_str}: {e}")
-                            continue
-
-                    if template_versions:
-                        templates[provider_model][template_name] = template_versions
-
-        return templates
-
-    def _is_valid_version(self, version_str: str) -> bool:
-        """Validate semantic versioning string"""
-        try:
-            version.parse(version_str)
-            return True
-        except version.InvalidVersion:
-            return False
-
-    def get_template(self,
-                     provider_model: str,
-                     template_name: str,
-                     template_version: Optional[str] = None) -> PromptTemplate:
-        """
-        Get a specific template version. If version not specified,
-        returns the latest version.
-        """
-        if provider_model not in self._templates:
-            raise ValueError(f"Unknown provider.model: {provider_model}")
-
-        if template_name not in self._templates[provider_model]:
-            raise ValueError(f"Unknown template: {template_name}")
-
-        versions = self._templates[provider_model][template_name]
-
-        if template_version:
-            if template_version not in versions:
-                raise ValueError(f"Template version {template_version} not found")
-            return versions[template_version]
-
-        # Return latest version
-        latest = max(versions.keys(), key=version.parse)
-        return versions[latest]
-
-    def list_templates(self, provider_model: str) -> Dict[str, list]:
-        """
-        List all available templates and their versions for a provider.model
-        Returns: {template_name: [version1, version2, ...]}
-        """
-        if provider_model not in self._templates:
-            raise ValueError(f"Unknown provider.model: {provider_model}")
-
-        return {
-            template_name: sorted(versions.keys(), key=version.parse)
-            for template_name, versions in self._templates[provider_model].items()
-        }

common/models/document.py

@@ -8,9 +8,10 @@ import sqlalchemy as sa
 
 class Catalog(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    name = db.Column(db.String(50), nullable=False)
+    name = db.Column(db.String(50), nullable=False, unique=True)
     description = db.Column(db.Text, nullable=True)
     type = db.Column(db.String(50), nullable=False, default="STANDARD_CATALOG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
 
     min_chunk_size = db.Column(db.Integer, nullable=True, default=1500)
     max_chunk_size = db.Column(db.Integer, nullable=True, default=2500)
@@ -26,6 +27,20 @@ class Catalog(db.Model):
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
     updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'description': self.description,
+            'type': self.type,
+            'type_version': self.type_version,
+            'min_chunk_size': self.min_chunk_size,
+            'max_chunk_size': self.max_chunk_size,
+            'user_metadata': self.user_metadata,
+            'system_metadata': self.system_metadata,
+            'configuration': self.configuration,
+        }
+
 
 class Processor(db.Model):
     id = db.Column(db.Integer, primary_key=True)
@@ -34,6 +49,7 @@ class Processor(db.Model):
     catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True)
     type = db.Column(db.String(50), nullable=False)
     sub_file_type = db.Column(db.String(50), nullable=True)
+    active = db.Column(db.Boolean, nullable=True, default=True)
 
     # Tuning enablers
     tuning = db.Column(db.Boolean, nullable=True, default=False)
@@ -89,6 +105,12 @@ class Document(db.Model):
     # Relations
     versions = db.relationship('DocumentVersion', backref='document', lazy=True)
 
+    @property
+    def latest_version(self):
+        """Returns the latest document version (the one with highest id)"""
+        from sqlalchemy import desc
+        return DocumentVersion.query.filter_by(doc_id=self.id).order_by(desc(DocumentVersion.id)).first()
+
     def __repr__(self):
         return f"<Document {self.id}: {self.name}>"
 

common/models/entitlements.py

@@ -1,4 +1,14 @@
+from sqlalchemy.sql.expression import text
+
 from common.extensions import db
+from datetime import datetime as dt, timezone as tz
+from enum import Enum
+from sqlalchemy import event
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.hybrid import hybrid_property
+from dateutil.relativedelta import relativedelta
+
+from common.utils.database import Database
 
 
 class BusinessEventLog(db.Model):
@@ -25,6 +35,7 @@ class BusinessEventLog(db.Model):
     llm_metrics_prompt_tokens = db.Column(db.Integer)
     llm_metrics_completion_tokens = db.Column(db.Integer)
     llm_metrics_total_time = db.Column(db.Float)
+    llm_metrics_nr_of_pages = db.Column(db.Integer)
     llm_metrics_call_count = db.Column(db.Integer)
     llm_interaction_type = db.Column(db.String(20))
     message = db.Column(db.Text)
@@ -41,6 +52,7 @@ class License(db.Model):
     tier_id = db.Column(db.Integer, db.ForeignKey('public.license_tier.id'),nullable=False)  # 'small', 'medium', 'custom'
     start_date = db.Column(db.Date, nullable=False)
     end_date = db.Column(db.Date, nullable=True)
+    nr_of_periods = db.Column(db.Integer, nullable=False)
     currency = db.Column(db.String(20), nullable=False)
     yearly_payment = db.Column(db.Boolean, nullable=False, default=False)
     basic_fee = db.Column(db.Float, nullable=False)
@@ -55,10 +67,41 @@ class License(db.Model):
     additional_interaction_bucket = db.Column(db.Integer, nullable=False)
     overage_embedding = db.Column(db.Float, nullable=False, default=0)
     overage_interaction = db.Column(db.Float, nullable=False, default=0)
+    additional_storage_allowed = db.Column(db.Boolean, nullable=False, default=False)
+    additional_embedding_allowed = db.Column(db.Boolean, nullable=False, default=False)
+    additional_interaction_allowed = db.Column(db.Boolean, nullable=False, default=False)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
 
     tenant = db.relationship('Tenant', back_populates='licenses')
     license_tier = db.relationship('LicenseTier', back_populates='licenses')
-    usages = db.relationship('LicenseUsage', order_by='LicenseUsage.period_start_date', back_populates='license')
+    periods = db.relationship('LicensePeriod', back_populates='license',
+                              order_by='LicensePeriod.period_number',
+                              cascade='all, delete-orphan')
+
+def calculate_end_date(start_date, nr_of_periods):
+    """Utility functie om einddatum te berekenen"""
+    if start_date and nr_of_periods:
+        return start_date + relativedelta(months=nr_of_periods) - relativedelta(days=1)
+    return None
+
+# Luister naar start_date wijzigingen
+@event.listens_for(License.start_date, 'set')
+def set_start_date(target, value, oldvalue, initiator):
+    """Bijwerken van end_date wanneer start_date wordt aangepast"""
+    if value and target.nr_of_periods:
+        target.end_date = calculate_end_date(value, target.nr_of_periods)
+
+# Luister naar nr_of_periods wijzigingen
+@event.listens_for(License.nr_of_periods, 'set')
+def set_nr_of_periods(target, value, oldvalue, initiator):
+    """Bijwerken van end_date wanneer nr_of_periods wordt aangepast"""
+    if value and target.start_date:
+        target.end_date = calculate_end_date(target.start_date, value)
 
 
 class LicenseTier(db.Model):
@@ -87,7 +130,219 @@ class LicenseTier(db.Model):
     standard_overage_embedding = db.Column(db.Float, nullable=False, default=0)
     standard_overage_interaction = db.Column(db.Float, nullable=False, default=0)
 
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
     licenses = db.relationship('License', back_populates='license_tier')
+    partner_services = db.relationship('PartnerServiceLicenseTier', back_populates='license_tier')
+
+
+class PartnerServiceLicenseTier(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    partner_service_id = db.Column(db.Integer, db.ForeignKey('public.partner_service.id'), primary_key=True,
+                                   nullable=False)
+    license_tier_id = db.Column(db.Integer, db.ForeignKey('public.license_tier.id'), primary_key=True,
+                                nullable=False)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    license_tier = db.relationship('LicenseTier', back_populates='partner_services')
+    partner_service = db.relationship('PartnerService', back_populates='license_tiers')
+
+
+class PeriodStatus(Enum):
+    UPCOMING = "UPCOMING"   # The period is still in the future
+    PENDING = "PENDING"     # The period is active, but prepaid is not yet received
+    ACTIVE = "ACTIVE"       # The period is active and prepaid has been received
+    COMPLETED = "COMPLETED" # The period has been completed, but not yet invoiced
+    INVOICED = "INVOICED"   # The period has been completed and invoiced, but overage payment still pending
+    CLOSED = "CLOSED"       # The period has been closed, invoiced and fully paid
+
+
+class LicensePeriod(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+    # Period identification
+    period_number = db.Column(db.Integer, nullable=False)
+    period_start = db.Column(db.Date, nullable=False)
+    period_end = db.Column(db.Date, nullable=False)
+
+    # License configuration snapshot - copied from license when period is created
+    currency = db.Column(db.String(20), nullable=True)
+    basic_fee = db.Column(db.Float, nullable=True)
+    max_storage_mb = db.Column(db.Integer, nullable=True)
+    additional_storage_price = db.Column(db.Float, nullable=True)
+    additional_storage_bucket = db.Column(db.Integer, nullable=True)
+    included_embedding_mb = db.Column(db.Integer, nullable=True)
+    additional_embedding_price = db.Column(db.Numeric(10, 4), nullable=True)
+    additional_embedding_bucket = db.Column(db.Integer, nullable=True)
+    included_interaction_tokens = db.Column(db.Integer, nullable=True)
+    additional_interaction_token_price = db.Column(db.Numeric(10, 4), nullable=True)
+    additional_interaction_bucket = db.Column(db.Integer, nullable=True)
+    
+    # Allowance flags - can be changed from False to True within a period
+    additional_storage_allowed = db.Column(db.Boolean, nullable=True, default=False)
+    additional_embedding_allowed = db.Column(db.Boolean, nullable=True, default=False)
+    additional_interaction_allowed = db.Column(db.Boolean, nullable=True, default=False)
+
+    # Status tracking
+    status = db.Column(db.Enum(PeriodStatus), nullable=False, default=PeriodStatus.UPCOMING)
+
+    # State transition timestamps
+    upcoming_at = db.Column(db.DateTime, nullable=True)
+    pending_at = db.Column(db.DateTime, nullable=True)
+    active_at = db.Column(db.DateTime, nullable=True)
+    completed_at = db.Column(db.DateTime, nullable=True)
+    invoiced_at = db.Column(db.DateTime, nullable=True)
+    closed_at = db.Column(db.DateTime, nullable=True)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relationships
+    license = db.relationship('License', back_populates='periods')
+    license_usage = db.relationship('LicenseUsage',
+                                    uselist=False,  # This makes it one-to-one
+                                    back_populates='license_period',
+                                    cascade='all, delete-orphan')
+    payments = db.relationship('Payment', back_populates='license_period')
+    invoices = db.relationship('Invoice', back_populates='license_period',
+                               cascade='all, delete-orphan')
+
+    def update_allowance(self, allowance_type, allow_value, user_id=None):
+        """
+        Update an allowance flag within a period
+        Only allows transitioning from False to True
+        
+        Args:
+            allowance_type: One of 'storage', 'embedding', or 'interaction'
+            allow_value: The new value (must be True)
+            user_id: User ID performing the update
+            
+        Raises:
+            ValueError: If trying to change from True to False, or invalid allowance type
+        """
+        field_name = f"additional_{allowance_type}_allowed"
+        
+        # Verify valid field
+        if not hasattr(self, field_name):
+            raise ValueError(f"Invalid allowance type: {allowance_type}")
+            
+        # Get current value
+        current_value = getattr(self, field_name)
+        
+        # Only allow False -> True transition
+        if current_value is True and allow_value is True:
+            # Already True, no change needed
+            return
+        elif allow_value is False:
+            raise ValueError(f"Cannot change {field_name} from {current_value} to False")
+        
+        # Update the field
+        setattr(self, field_name, True)
+        self.updated_at = dt.now(tz.utc)
+        if user_id:
+            self.updated_by = user_id
+
+    @property
+    def prepaid_invoice(self):
+        """Get the prepaid invoice for this period"""
+        return Invoice.query.filter_by(
+            license_period_id=self.id,
+            invoice_type=PaymentType.PREPAID
+        ).first()
+
+    @property
+    def overage_invoice(self):
+        """Get the overage invoice for this period"""
+        return Invoice.query.filter_by(
+            license_period_id=self.id,
+            invoice_type=PaymentType.POSTPAID
+        ).first()
+
+    @property
+    def prepaid_payment(self):
+        """Get the prepaid payment for this period"""
+        return Payment.query.filter_by(
+            license_period_id=self.id,
+            payment_type=PaymentType.PREPAID
+        ).first()
+
+    @property
+    def overage_payment(self):
+        """Get the overage payment for this period"""
+        return Payment.query.filter_by(
+            license_period_id=self.id,
+            payment_type=PaymentType.POSTPAID
+        ).first()
+
+    @property
+    def all_invoices(self):
+        """Get all invoices for this period"""
+        return self.invoices
+
+    @property
+    def all_payments(self):
+        """Get all payments for this period"""
+        return self.payments
+
+    def transition_status(self, new_status: PeriodStatus, user_id: int = None):
+        """Transition to a new status with proper validation and logging"""
+        if not self.can_transition_to(new_status):
+            raise ValueError(f"Invalid status transition from {self.status} to {new_status}")
+
+        self.status = new_status
+        self.updated_at = dt.now(tz.utc)
+        if user_id:
+            self.updated_by = user_id
+
+        # Set appropriate timestamps
+        if new_status == PeriodStatus.ACTIVE and not self.prepaid_received_at:
+            self.prepaid_received_at = dt.now(tz.utc)
+        elif new_status == PeriodStatus.COMPLETED:
+            self.completed_at = dt.now(tz.utc)
+        elif new_status == PeriodStatus.INVOICED:
+            self.invoiced_at = dt.now(tz.utc)
+        elif new_status == PeriodStatus.CLOSED:
+            self.closed_at = dt.now(tz.utc)
+
+    @property
+    def is_overdue(self):
+        """Check if a prepaid payment is overdue"""
+        return (self.status == PeriodStatus.PENDING and
+                self.period_start <= dt.now(tz.utc).date())
+
+    def can_transition_to(self, new_status: PeriodStatus) -> bool:
+        """Check if a status transition is valid"""
+        valid_transitions = {
+            PeriodStatus.UPCOMING: [PeriodStatus.ACTIVE, PeriodStatus.PENDING],
+            PeriodStatus.PENDING: [PeriodStatus.ACTIVE],
+            PeriodStatus.ACTIVE: [PeriodStatus.COMPLETED],
+            PeriodStatus.COMPLETED: [PeriodStatus.INVOICED, PeriodStatus.CLOSED],
+            PeriodStatus.INVOICED: [PeriodStatus.CLOSED],
+            PeriodStatus.CLOSED: []
+        }
+        return new_status in valid_transitions.get(self.status, [])
+
+    def __repr__(self):
+        return f'<LicensePeriod {self.id}: License {self.license_id}, Period {self.period_number}>'
 
 
 class LicenseUsage(db.Model):
@@ -95,7 +350,6 @@ class LicenseUsage(db.Model):
     __table_args__ = {'schema': 'public'}
 
     id = db.Column(db.Integer, primary_key=True)
-    license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
     tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
     storage_mb_used = db.Column(db.Float, default=0)
     embedding_mb_used = db.Column(db.Float, default=0)
@@ -105,9 +359,170 @@ class LicenseUsage(db.Model):
     interaction_prompt_tokens_used = db.Column(db.Integer, default=0)
     interaction_completion_tokens_used = db.Column(db.Integer, default=0)
     interaction_total_tokens_used = db.Column(db.Integer, default=0)
-    period_start_date = db.Column(db.Date, nullable=False)
-    period_end_date = db.Column(db.Date, nullable=False)
+    license_period_id = db.Column(db.Integer, db.ForeignKey('public.license_period.id'), nullable=False)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    license_period = db.relationship('LicensePeriod', back_populates='license_usage')
+
+    def recalculate_storage(self):
+        Database(self.tenant_id).switch_schema()
+        # Perform a SUM operation to get the total file size from document_versions
+        total_storage = db.session.execute(text(f"""
+                SELECT SUM(file_size) 
+                FROM document_version
+            """)).scalar()
+
+        self.storage_mb_used = total_storage
+
+
+class PaymentType(Enum):
+    PREPAID = "PREPAID"
+    POSTPAID = "POSTPAID"
+
+
+class PaymentStatus(Enum):
+    PENDING = "PENDING"
+    PAID = "PAID"
+    FAILED = "FAILED"
+    CANCELLED = "CANCELLED"
+
+
+class Payment(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_period_id = db.Column(db.Integer, db.ForeignKey('public.license_period.id'), nullable=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+    # Payment details
+    payment_type = db.Column(db.Enum(PaymentType), nullable=False)
+    amount = db.Column(db.Numeric(10, 2), nullable=False)
+    currency = db.Column(db.String(3), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+
+    # Status tracking
+    status = db.Column(db.Enum(PaymentStatus), nullable=False, default=PaymentStatus.PENDING)
+
+    # External provider information
+    external_payment_id = db.Column(db.String(255), nullable=True)
+    payment_method = db.Column(db.String(50), nullable=True)  # credit_card, bank_transfer, etc.
+    provider_data = db.Column(JSONB, nullable=True)  # Provider-specific data
+
+    # Payment information
+    paid_at = db.Column(db.DateTime, nullable=True)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relationships
+    license_period = db.relationship('LicensePeriod', back_populates='payments')
+    invoice = db.relationship('Invoice', back_populates='payment', uselist=False)
+
+    @property
+    def is_overdue(self):
+        """Check if payment is overdue"""
+        if self.status != PaymentStatus.PENDING:
+            return False
+
+        # For prepaid payments, check if period start has passed
+        if (self.payment_type == PaymentType.PREPAID and
+                self.license_period_id):
+            return self.license_period.period_start <= dt.now(tz.utc).date()
+
+        # For postpaid, check against due date (would be on invoice)
+        return False
+
+    def __repr__(self):
+        return f'<Payment {self.id}: {self.payment_type} {self.amount} {self.currency}>'
+
+
+class InvoiceStatus(Enum):
+    DRAFT = "DRAFT"
+    SENT = "SENT"
+    PAID = "PAID"
+    OVERDUE = "OVERDUE"
+    CANCELLED = "CANCELLED"
+
+
+class Invoice(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_period_id = db.Column(db.Integer, db.ForeignKey('public.license_period.id'), nullable=False)
+    payment_id = db.Column(db.Integer, db.ForeignKey('public.payment.id'), nullable=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+    # Invoice details
+    invoice_type = db.Column(db.Enum(PaymentType), nullable=False)
+    invoice_number = db.Column(db.String(50), unique=True, nullable=False)
+    invoice_date = db.Column(db.Date, nullable=False)
+    due_date = db.Column(db.Date, nullable=False)
+
+    # Financial details
+    amount = db.Column(db.Numeric(10, 2), nullable=False)
+    currency = db.Column(db.String(3), nullable=False)
+    tax_amount = db.Column(db.Numeric(10, 2), default=0)
+
+    # Descriptive fields
+    description = db.Column(db.Text, nullable=True)
+    status = db.Column(db.Enum(InvoiceStatus), nullable=False, default=InvoiceStatus.DRAFT)
+
+    # Timestamps
+    sent_at = db.Column(db.DateTime, nullable=True)
+    paid_at = db.Column(db.DateTime, nullable=True)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relationships
+    license_period = db.relationship('LicensePeriod', back_populates='invoices')
+    payment = db.relationship('Payment', back_populates='invoice')
+
+    def __repr__(self):
+        return f'<Invoice {self.invoice_number}: {self.amount} {self.currency}>'
+
+
+class LicenseChangeLog(db.Model):
+    """
+    Log of changes to license configurations
+    Used for auditing and tracking when/why license details changed
+    """
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+    
+    id = db.Column(db.Integer, primary_key=True)
+    license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
+    changed_at = db.Column(db.DateTime, nullable=False, default=lambda: dt.now(tz.utc))
+    
+    # What changed
+    field_name = db.Column(db.String(100), nullable=False)
+    old_value = db.Column(db.String(255), nullable=True)
+    new_value = db.Column(db.String(255), nullable=False)
+    
+    # Why it changed
+    reason = db.Column(db.Text, nullable=True)
+    
+    # Standard audit fields
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    
+    # Relationships
+    license = db.relationship('License', backref=db.backref('change_logs', order_by='LicenseChangeLog.changed_at'))
+    
+    def __repr__(self):
+        return f'<LicenseChangeLog: {self.license_id} {self.field_name} {self.old_value} -> {self.new_value}>'
 
-    license = db.relationship('License', back_populates='usages')
 
 

common/models/interaction.py

@@ -1,14 +1,14 @@
 from sqlalchemy.dialects.postgresql import JSONB
 
 from ..extensions import db
-from .user import User, Tenant
+from .user import User, Tenant, TenantMake
 from .document import Embedding, Retriever
 
 
 class ChatSession(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     user_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
-    session_id = db.Column(db.String(36), nullable=True)
+    session_id = db.Column(db.String(49), nullable=True)
     session_start = db.Column(db.DateTime, nullable=False)
     session_end = db.Column(db.DateTime, nullable=True)
     timezone = db.Column(db.String(30), nullable=True)
@@ -29,6 +29,7 @@ class Specialist(db.Model):
     tuning = db.Column(db.Boolean, nullable=True, default=False)
     configuration = db.Column(JSONB, nullable=True)
     arguments = db.Column(JSONB, nullable=True)
+    active = db.Column(db.Boolean, nullable=True, default=True)
 
     # Relationship to retrievers through the association table
     retrievers = db.relationship('SpecialistRetriever', backref='specialist', lazy=True,
@@ -44,6 +45,21 @@ class Specialist(db.Model):
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
     updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
+    def __repr__(self):
+        return f"<Specialist {self.id}: {self.name}>"
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'description': self.description,
+            'type': self.type,
+            'type_version': self.type_version,
+            'configuration': self.configuration,
+            'arguments': self.arguments,
+            'active': self.active,
+        }
+
 
 class EveAIAsset(db.Model):
     id = db.Column(db.Integer, primary_key=True)
@@ -51,25 +67,23 @@ class EveAIAsset(db.Model):
     description = db.Column(db.Text, nullable=True)
     type = db.Column(db.String(50), nullable=False, default="DOCUMENT_TEMPLATE")
     type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
-    valid_from = db.Column(db.DateTime, nullable=True)
-    valid_to = db.Column(db.DateTime, nullable=True)
 
-    # Versioning Information
-    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
-    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
-    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
-    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
-
-    # Relations
-    versions = db.relationship('EveAIAssetVersion', backref='asset', lazy=True)
-
-
-class EveAIAssetVersion(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    asset_id = db.Column(db.Integer, db.ForeignKey(EveAIAsset.id), nullable=False)
+    # Storage information
     bucket_name = db.Column(db.String(255), nullable=True)
+    object_name = db.Column(db.String(200), nullable=True)
+    file_type = db.Column(db.String(20), nullable=True)
+    file_size = db.Column(db.Float, nullable=True)
+
+    # Metadata information
+    user_metadata = db.Column(JSONB, nullable=True)
+    system_metadata = db.Column(JSONB, nullable=True)
+
+    # Configuration information
     configuration = db.Column(JSONB, nullable=True)
-    arguments = db.Column(JSONB, nullable=True)
+
+    # Cost information
+    prompt_tokens = db.Column(db.Integer, nullable=True)
+    completion_tokens = db.Column(db.Integer, nullable=True)
 
     # Versioning Information
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
@@ -77,25 +91,25 @@ class EveAIAssetVersion(db.Model):
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
     updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
-    # Relations
-    instructions = db.relationship('EveAIAssetInstruction', backref='asset_version', lazy=True)
+    last_used_at = db.Column(db.DateTime, nullable=True)
 
 
-class EveAIAssetInstruction(db.Model):
+class EveAIDataCapsule(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    asset_version_id = db.Column(db.Integer, db.ForeignKey(EveAIAssetVersion.id), nullable=False)
-    name = db.Column(db.String(255), nullable=False)
-    content = db.Column(db.Text, nullable=True)
+    chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=False)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    configuration = db.Column(JSONB, nullable=True)
+    data = db.Column(JSONB, nullable=True)
 
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
-class EveAIProcessedAsset(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    asset_version_id = db.Column(db.Integer, db.ForeignKey(EveAIAssetVersion.id), nullable=False)
-    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=True)
-    chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=True)
-    bucket_name = db.Column(db.String(255), nullable=True)
-    object_name = db.Column(db.String(255), nullable=True)
-    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    # Unieke constraint voor chat_session_id, type en type_version
+    __table_args__ = (db.UniqueConstraint('chat_session_id', 'type', 'type_version', name='uix_data_capsule_session_type_version'),)
 
 
 class EveAIAgent(db.Model):
@@ -189,6 +203,7 @@ class Interaction(db.Model):
     question_at = db.Column(db.DateTime, nullable=False)
     detailed_question_at = db.Column(db.DateTime, nullable=True)
     answer_at = db.Column(db.DateTime, nullable=True)
+    processing_error = db.Column(db.String(255), nullable=True)
 
     # Relations
     embeddings = db.relationship('InteractionEmbedding', backref='interaction', lazy=True)
@@ -214,3 +229,36 @@ class SpecialistDispatcher(db.Model):
     dispatcher_id = db.Column(db.Integer, db.ForeignKey(Dispatcher.id, ondelete='CASCADE'), primary_key=True)
 
     dispatcher = db.relationship("Dispatcher", backref="specialist_dispatchers")
+
+
+class SpecialistMagicLink(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), nullable=False)
+    tenant_make_id = db.Column(db.Integer, db.ForeignKey(TenantMake.id, ondelete='CASCADE'), nullable=True)
+    magic_link_code = db.Column(db.String(55), nullable=False, unique=True)
+
+    valid_from = db.Column(db.DateTime, nullable=True)
+    valid_to = db.Column(db.DateTime, nullable=True)
+
+    specialist_args = db.Column(JSONB, nullable=True)
+
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+    def __repr__(self):
+        return f"<SpecialistMagicLink {self.specialist_id} {self.magic_link_code}>"
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'description': self.description,
+            'magic_link_code': self.magic_link_code,
+            'valid_from': self.valid_from,
+            'valid_to': self.valid_to,
+            'specialist_args': self.specialist_args,
+        }

common/models/user.py

@@ -2,7 +2,7 @@ from datetime import date
 
 from common.extensions import db
 from flask_security import UserMixin, RoleMixin
-from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 import sqlalchemy as sa
 
 from common.models.entitlements import License
@@ -20,28 +20,24 @@ class Tenant(db.Model):
 
     # company Information
     id = db.Column(db.Integer, primary_key=True)
+    code = db.Column(db.String(50), unique=True, nullable=True)
     name = db.Column(db.String(80), unique=True, nullable=False)
     website = db.Column(db.String(255), nullable=True)
     timezone = db.Column(db.String(50), nullable=True, default='UTC')
-    rag_context = db.Column(db.Text, nullable=True)
     type = db.Column(db.String(20), nullable=True, server_default='Active')
 
-    # language information
-    default_language = db.Column(db.String(2), nullable=True)
-    allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True)
-
-    # LLM specific choices
-    llm_model = db.Column(db.String(50), nullable=True)
-
-    # Entitlements
+   # Entitlements
     currency = db.Column(db.String(20), nullable=True)
     storage_dirty = db.Column(db.Boolean, nullable=True, default=False)
+    default_tenant_make_id = db.Column(db.Integer, db.ForeignKey('public.tenant_make.id'), nullable=True)
 
     # Relations
     users = db.relationship('User', backref='tenant')
     domains = db.relationship('TenantDomain', backref='tenant')
     licenses = db.relationship('License', back_populates='tenant')
     license_usages = db.relationship('LicenseUsage', backref='tenant')
+    tenant_makes = db.relationship('TenantMake', backref='tenant', foreign_keys='TenantMake.tenant_id')
+    default_tenant_make = db.relationship('TenantMake', foreign_keys=[default_tenant_make_id], uselist=False)
 
     @property
     def current_license(self):
@@ -61,12 +57,9 @@ class Tenant(db.Model):
             'name': self.name,
             'website': self.website,
             'timezone': self.timezone,
-            'rag_context': self.rag_context,
             'type': self.type,
-            'default_language': self.default_language,
-            'allowed_languages': self.allowed_languages,
-            'llm_model': self.llm_model,
             'currency': self.currency,
+            'default_tenant_make_id': self.default_tenant_make_id,
         }
 
 
@@ -99,6 +92,7 @@ class User(db.Model, UserMixin):
 
     # User Information
     id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
     user_name = db.Column(db.String(80), unique=True, nullable=False)
     email = db.Column(db.String(255), unique=True, nullable=False)
     password = db.Column(db.String(255), nullable=True)
@@ -120,7 +114,6 @@ class User(db.Model, UserMixin):
 
     # Relations
     roles = db.relationship('Role', secondary=RolesUsers.__table__, backref=db.backref('users', lazy='dynamic'))
-    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
 
     def __repr__(self):
         return '<User %r>' % self.user_name
@@ -143,9 +136,9 @@ class TenantDomain(db.Model):
 
     # Versioning Information
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
-    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=False)
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=False)
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
-    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
 
     def __repr__(self):
         return f"<TenantDomain {self.id}: {self.domain}>"
@@ -176,3 +169,188 @@ class TenantProject(db.Model):
 
     def __repr__(self):
         return f"<TenantProject {self.id}: {self.name}>"
+
+
+class TenantMake(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    name = db.Column(db.String(50), nullable=False, unique=True)
+    description = db.Column(db.Text, nullable=True)
+    active = db.Column(db.Boolean, nullable=False, default=True)
+    website = db.Column(db.String(255), nullable=True)
+    logo_url = db.Column(db.String(255), nullable=True)
+    default_language = db.Column(db.String(2), nullable=True)
+    allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True)
+
+    # Chat customisation options
+    chat_customisation_options = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    def __repr__(self):
+        return f"<TenantMake {self.id} for tenant {self.tenant_id}: {self.name}>"
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'description': self.description,
+            'active': self.active,
+            'website': self.website,
+            'logo_url': self.logo_url,
+            'chat_customisation_options': self.chat_customisation_options,
+            'allowed_languages': self.allowed_languages,
+            'default_language': self.default_language,
+        }
+
+
+class Partner(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False, unique=True)
+    code = db.Column(db.String(50), unique=True, nullable=False)
+
+    # Basic information
+    logo_url = db.Column(db.String(255), nullable=True)
+    active = db.Column(db.Boolean, default=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    # Relationships
+    services = db.relationship('PartnerService', back_populates='partner')
+    tenant = db.relationship('Tenant', backref=db.backref('partner', uselist=False))
+
+    def to_dict(self):
+        services_info = []
+        for service in self.services:
+            services_info.append({
+                'id': service.id,
+                'name': service.name,
+                'description': service.description,
+                'type': service.type,
+                'type_version': service.type_version,
+                'active': service.active,
+                'configuration': service.configuration,
+                'permissions': service.permissions,
+            })
+        return {
+            'id': self.id,
+            'tenant_id': self.tenant_id,
+            'code': self.code,
+            'logo_url': self.logo_url,
+            'active': self.active,
+            'name': self.tenant.name,
+            'services': services_info
+        }
+
+
+class PartnerService(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    partner_id = db.Column(db.Integer, db.ForeignKey('public.partner.id'), nullable=False)
+
+    # Basic info
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+
+    # Service type with versioning (similar to your specialist/retriever pattern)
+    type = db.Column(db.String(50), nullable=False)  # REFERRAL, KNOWLEDGE, SPECIALIST, IMPLEMENTATION, WHITE_LABEL
+    type_version = db.Column(db.String(20), nullable=False, default="1.0.0")
+
+    # Status
+    active = db.Column(db.Boolean, default=True)
+
+    # Dynamic configuration specific to this service - using JSONB like your other models
+    configuration = db.Column(db.JSON, nullable=True)
+    permissions = db.Column(db.JSON, nullable=True)
+
+    # For services that need to track shared resources
+    system_metadata = db.Column(db.JSON, nullable=True)
+    user_metadata = db.Column(db.JSON, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    # Relationships
+    partner = db.relationship('Partner', back_populates='services')
+    license_tiers = db.relationship('PartnerServiceLicenseTier', back_populates='partner_service')
+
+
+class PartnerTenant(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    partner_service_id = db.Column(db.Integer, db.ForeignKey('public.partner_service.id'), primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), primary_key=True)
+
+    # JSONB for flexible configuration specific to this relationship
+    configuration = db.Column(db.JSON, nullable=True)
+
+    # Tracking
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+
+class SpecialistMagicLinkTenant(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    magic_link_code = db.Column(db.String(55), primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+
+class TranslationCache(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    cache_key = db.Column(db.String(16), primary_key=True)
+    source_text = db.Column(db.Text, nullable=False)
+    translated_text = db.Column(db.Text, nullable=False)
+    source_language = db.Column(db.String(2), nullable=True)
+    target_language = db.Column(db.String(2), nullable=False)
+    context = db.Column(db.Text, nullable=True)
+
+    # Translation cost
+    prompt_tokens = db.Column(db.Integer, nullable=False)
+    completion_tokens = db.Column(db.Integer, nullable=False)
+
+    # Tracking
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    last_used_at = db.Column(db.DateTime, nullable=True)
+
+
+class PartnerRAGRetriever(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('tenant_id', 'retriever_id'),
+        db.UniqueConstraint('partner_id', 'tenant_id', 'retriever_id'),
+        {'schema': 'public'},
+    )
+
+    partner_id = db.Column(db.Integer, db.ForeignKey('public.partner.id'), nullable=False)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    retriever_id = db.Column(db.Integer, nullable=False)

common/services/entitlements/__init__.py

@@ -0,0 +1,9 @@
+from common.services.entitlements.license_period_services import LicensePeriodServices
+from common.services.entitlements.license_usage_services import LicenseUsageServices
+from common.services.entitlements.license_tier_services import LicenseTierServices
+
+__all__ = [
+    'LicensePeriodServices',
+    'LicenseUsageServices',
+    'LicenseTierServices'
+]

common/services/entitlements/license_period_services.py

@@ -0,0 +1,247 @@
+from dateutil.relativedelta import relativedelta
+from datetime import datetime as dt, timezone as tz, timedelta
+from flask import current_app
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.expression import and_
+
+from common.extensions import db
+from common.models.entitlements import LicensePeriod, License, PeriodStatus, LicenseUsage
+from common.utils.eveai_exceptions import EveAILicensePeriodsExceeded, EveAIPendingLicensePeriod, EveAINoActiveLicense
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+
+
+class LicensePeriodServices:
+    @staticmethod
+    def find_current_license_period_for_usage(tenant_id: int) -> LicensePeriod:
+        """
+        Find the current license period for a tenant. It ensures the status of the different license periods are adapted
+        when required, and a LicenseUsage object is created if required.
+
+        Args:
+            tenant_id: The ID of the tenant to find the license period for
+
+        Raises:
+            EveAIException: and derived classes
+        """
+        try:
+            current_app.logger.debug(f"Finding current license period for tenant {tenant_id}")
+            current_date = dt.now(tz.utc).date()
+            license_period = (db.session.query(LicensePeriod)
+                              .filter_by(tenant_id=tenant_id)
+                              .filter(and_(LicensePeriod.period_start <= current_date,
+                                           LicensePeriod.period_end >= current_date))
+                              .first())
+            current_app.logger.debug(f"End searching for license period for tenant {tenant_id} ")
+            if not license_period:
+                current_app.logger.debug(f"No license period found for tenant {tenant_id} on date {current_date}")
+                license_period = LicensePeriodServices._create_next_license_period_for_usage(tenant_id)
+                current_app.logger.debug(f"Created license period {license_period.id} for tenant {tenant_id}")
+            if license_period:
+                current_app.logger.debug(f"Found license period {license_period.id} for tenant {tenant_id} "
+                                         f"with status {license_period.status}")
+                match license_period.status:
+                    case PeriodStatus.UPCOMING | PeriodStatus.PENDING:
+                        current_app.logger.debug(f"In upcoming state")
+                        LicensePeriodServices._complete_last_license_period(tenant_id=tenant_id)
+                        current_app.logger.debug(f"Completed last license period for tenant {tenant_id}")
+                        LicensePeriodServices._activate_license_period(license_period=license_period)
+                        current_app.logger.debug(f"Activated license period {license_period.id} for tenant {tenant_id}")
+                        if not license_period.license_usage:
+                            new_license_usage = LicenseUsage(
+                                tenant_id=tenant_id,
+                            )
+                            new_license_usage.license_period = license_period
+                            try:
+                                db.session.add(new_license_usage)
+                                db.session.commit()
+
+                            except SQLAlchemyError as e:
+                                db.session.rollback()
+                                current_app.logger.error(
+                                    f"Error creating new license usage for license period "
+                                    f"{license_period.id}: {str(e)}")
+                                raise e
+                        if license_period.status == PeriodStatus.ACTIVE:
+                            return license_period
+                        else:
+                            # Status is PENDING, so no prepaid payment received. There is no license period we can use.
+                            # We allow for a delay of 5 days before raising an exception.
+                            current_date = dt.now(tz.utc).date()
+                            delta = abs(current_date - license_period.period_start)
+                            if delta > timedelta(days=current_app.config.get('ENTITLEMENTS_MAX_PENDING_DAYS', 5)):
+                                raise EveAIPendingLicensePeriod()
+                            else:
+                                return license_period
+                    case PeriodStatus.ACTIVE:
+                        return license_period
+            else:
+                raise EveAILicensePeriodsExceeded(license_id=None)
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error finding current license period for tenant {tenant_id}: {str(e)}")
+            raise e
+        except Exception as e:
+            raise e
+
+    @staticmethod
+    def _create_next_license_period_for_usage(tenant_id) -> LicensePeriod:
+        """
+        Create a new period for this license using the current license configuration
+
+        Args:
+            tenant_id: The ID of the tenant to create the period for
+
+        Returns:
+            LicensePeriod: The newly created license period
+        """
+        current_date = dt.now(tz.utc).date()
+
+        # Zoek de actieve licentie voor deze tenant op de huidige datum
+        the_license = (db.session.query(License)
+                      .filter_by(tenant_id=tenant_id)
+                      .filter(License.start_date <= current_date)
+                      .filter(License.end_date >= current_date)
+                      .first())
+
+        if not the_license:
+            current_app.logger.error(f"No active license found for tenant {tenant_id} on date {current_date}")
+            raise EveAINoActiveLicense(tenant_id=tenant_id)
+        else:
+            current_app.logger.debug(f"Found active license {the_license.id} for tenant {tenant_id} "
+                                    f"on date {current_date}")
+
+        next_period_number = 1
+        if the_license.periods:
+            # If there are existing periods, get the next sequential number
+            next_period_number = max(p.period_number for p in the_license.periods) + 1
+        current_app.logger.debug(f"Next period number for tenant {tenant_id} is {next_period_number}")
+
+        if next_period_number > the_license.nr_of_periods:
+            raise EveAILicensePeriodsExceeded(license_id=the_license.id)
+
+        new_license_period = LicensePeriod(
+            license_id=the_license.id,
+            tenant_id=tenant_id,
+            period_number=next_period_number,
+            period_start=the_license.start_date + relativedelta(months=next_period_number-1),
+            period_end=the_license.start_date + relativedelta(months=next_period_number, days=-1),
+            status=PeriodStatus.UPCOMING,
+            upcoming_at=dt.now(tz.utc),
+        )
+        set_logging_information(new_license_period, dt.now(tz.utc))
+
+        try:
+            current_app.logger.debug(f"Creating next license period for tenant {tenant_id} ")
+            db.session.add(new_license_period)
+            db.session.commit()
+            current_app.logger.info(f"Created next license period for tenant {tenant_id} "
+                                    f"with id {new_license_period.id}")
+            return new_license_period
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error creating next license period for tenant {tenant_id}: {str(e)}")
+            raise e
+
+    @staticmethod
+    def _activate_license_period(license_period_id: int = None, license_period: LicensePeriod = None) -> LicensePeriod:
+        """
+        Activate a license period
+
+        Args:
+            license_period_id: The ID of the license period to activate (optional if license_period is provided)
+            license_period: The LicensePeriod object to activate (optional if license_period_id is provided)
+
+        Returns:
+            LicensePeriod: The activated license period object
+
+        Raises:
+            ValueError: If neither license_period_id nor license_period is provided
+        """
+        current_app.logger.debug(f"Activating license period")
+        if license_period is None and license_period_id is None:
+            raise ValueError("Either license_period_id or license_period must be provided")
+
+        # Get a license period object if only ID was provided
+        if license_period is None:
+            current_app.logger.debug(f"Getting license period {license_period_id} to activate")
+            license_period = LicensePeriod.query.get_or_404(license_period_id)
+
+        if license_period.pending_at is not None:
+            license_period.pending_at = dt.now(tz.utc)
+        license_period.status = PeriodStatus.PENDING
+        if license_period.prepaid_payment:
+            # There is a payment received for the given period
+            license_period.active_at = dt.now(tz.utc)
+            license_period.status = PeriodStatus.ACTIVE
+
+        # Copy snapshot fields from the license to the period
+        the_license = License.query.get_or_404(license_period.license_id)
+        license_period.currency = the_license.currency
+        license_period.basic_fee = the_license.basic_fee
+        license_period.max_storage_mb = the_license.max_storage_mb
+        license_period.additional_storage_price = the_license.additional_storage_price
+        license_period.additional_storage_bucket = the_license.additional_storage_bucket
+        license_period.included_embedding_mb = the_license.included_embedding_mb
+        license_period.additional_embedding_price = the_license.additional_embedding_price
+        license_period.additional_embedding_bucket = the_license.additional_embedding_bucket
+        license_period.included_interaction_tokens = the_license.included_interaction_tokens
+        license_period.additional_interaction_token_price = the_license.additional_interaction_token_price
+        license_period.additional_interaction_bucket = the_license.additional_interaction_bucket
+        license_period.additional_storage_allowed = the_license.additional_storage_allowed
+        license_period.additional_embedding_allowed = the_license.additional_embedding_allowed
+        license_period.additional_interaction_allowed = the_license.additional_interaction_allowed
+
+        update_logging_information(license_period, dt.now(tz.utc))
+
+        if not license_period.license_usage:
+            license_period.license_usage = LicenseUsage(
+                tenant_id=license_period.tenant_id,
+                license_period_id=license_period.id,
+            )
+
+        license_period.license_usage.recalculate_storage()
+
+        try:
+            db.session.add(license_period)
+            db.session.add(license_period.license_usage)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error activating license period {license_period_id}: {str(e)}")
+            raise e
+
+        return license_period
+
+    @staticmethod
+    def _complete_last_license_period(tenant_id) -> None:
+        """
+        Complete the active or pending license period for a tenant. This is done by setting the status to COMPLETED.
+
+        Args:
+            tenant_id: De ID van de tenant
+        """
+        # Zoek de licenseperiode voor deze tenant met status ACTIVE of PENDING
+        active_period = (
+            db.session.query(LicensePeriod)
+            .filter_by(tenant_id=tenant_id)
+            .filter(LicensePeriod.status.in_([PeriodStatus.ACTIVE, PeriodStatus.PENDING]))
+            .first()
+        )
+
+        # Als er geen actieve periode gevonden is, hoeven we niets te doen
+        if not active_period:
+            return
+
+        # Zet de gevonden periode op COMPLETED
+        active_period.status = PeriodStatus.COMPLETED
+        active_period.completed_at = dt.now(tz.utc)
+        update_logging_information(active_period, dt.now(tz.utc))
+
+        try:
+            db.session.add(active_period)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error completing period {active_period.id} for {tenant_id}: {str(e)}")
+            raise e

common/services/entitlements/license_tier_services.py

@@ -0,0 +1,67 @@
+from flask import session, flash, current_app
+from datetime import datetime as dt, timezone as tz
+
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db
+from common.models.entitlements import PartnerServiceLicenseTier
+from common.models.user import Partner
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService, EveAINoSessionPartner
+from common.utils.model_logging_utils import set_logging_information
+
+
+class LicenseTierServices:
+    @staticmethod
+    def associate_license_tier_with_partner(license_tier_id):
+        """Associate a license tier with a partner"""
+        try:
+            partner_id = session['partner']['id']
+            # Get partner service (MANAGEMENT_SERVICE type)
+            partner = Partner.query.get(partner_id)
+            if not partner:
+                raise EveAINoSessionPartner()
+
+            # Find a management service for this partner
+            management_service = next((service for service in session['partner']['services']
+                                       if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+
+            if not management_service:
+                flash("Cannot associate license tier with partner. No management service defined for partner", "danger")
+                current_app.logger.error(f"No Management Service defined for partner {partner_id}"
+                                         f"trying to associate license tier {license_tier_id}.")
+                raise EveAINoManagementPartnerService()
+            # Check if the association already exists
+            existing_association = PartnerServiceLicenseTier.query.filter_by(
+                partner_service_id=management_service['id'],
+                license_tier_id=license_tier_id
+            ).first()
+
+            if existing_association:
+                # Association already exists, nothing to do
+                flash("License tier was already associated with partner", "info")
+                current_app.logger.info(f"Association between partner service {management_service['id']} and "
+                                        f"license tier {license_tier_id} already exists.")
+                return
+
+            # Create the association
+            association = PartnerServiceLicenseTier(
+                partner_service_id=management_service['id'],
+                license_tier_id=license_tier_id
+            )
+            set_logging_information(association, dt.now(tz.utc))
+
+            db.session.add(association)
+            db.session.commit()
+
+            flash("Successfully associated license tier to partner", "success")
+            current_app.logger.info(f"Successfully associated license tier {license_tier_id} with "
+                                    f"partner service {management_service['id']}")
+
+            return True
+
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash("Failed to associated license tier with partner service due to an internal error. "
+                  "Please contact the System Administrator", "danger")
+            current_app.logger.error(f"Error associating license tier {license_tier_id} with partner: {str(e)}")
+            raise e

common/services/entitlements/license_usage_services.py

@@ -0,0 +1,143 @@
+from dateutil.relativedelta import relativedelta
+from flask import session, current_app, flash
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.expression import text
+
+from common.extensions import db, cache_manager
+from common.models.entitlements import PartnerServiceLicenseTier, License, LicenseUsage, LicensePeriod, PeriodStatus
+from common.models.user import Partner, PartnerTenant
+from common.services.entitlements import LicensePeriodServices
+from common.utils.database import Database
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService, EveAINoActiveLicense, \
+    EveAIStorageQuotaExceeded, EveAIEmbeddingQuotaExceeded, EveAIInteractionQuotaExceeded, EveAILicensePeriodsExceeded, \
+    EveAIException
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+from datetime import datetime as dt, timezone as tz
+
+from common.utils.security_utils import current_user_has_role
+
+
+class LicenseUsageServices:
+    @staticmethod
+    def check_storage_and_embedding_quota(tenant_id: int, file_size_mb: float) -> None:
+        """
+        Check if a tenant can add a new document without exceeding storage and embedding quotas
+
+        Args:
+            tenant_id: ID of the tenant
+            file_size_mb: Size of the file in MB
+
+        Raises:
+            EveAIStorageQuotaExceeded: If storage quota would be exceeded
+            EveAIEmbeddingQuotaExceeded: If embedding quota would be exceeded
+            EveAINoActiveLicense: If no active license is found
+            EveAIException: For other errors
+        """
+        # Get active license period
+        license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
+        # Early return if both overruns are allowed - no need to check usage at all
+        if license_period.additional_storage_allowed and license_period.additional_embedding_allowed:
+            return
+
+        # Check storage quota only if overruns are not allowed
+        if not license_period.additional_storage_allowed:
+            LicenseUsageServices._validate_storage_quota(license_period, file_size_mb)
+
+        # Check embedding quota only if overruns are not allowed
+        if not license_period.additional_embedding_allowed:
+            LicenseUsageServices._validate_embedding_quota(license_period, file_size_mb)
+
+    @staticmethod
+    def check_embedding_quota(tenant_id: int, file_size_mb: float) -> None:
+        """
+        Check if a tenant can re-embed a document without exceeding embedding quota
+
+        Args:
+            tenant_id: ID of the tenant
+            file_size_mb: Size of the file in MB
+
+        Raises:
+            EveAIEmbeddingQuotaExceeded: If embedding quota would be exceeded
+            EveAINoActiveLicense: If no active license is found
+            EveAIException: For other errors
+        """
+        # Get active license period
+        license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
+        # Early return if both overruns are allowed - no need to check usage at all
+        if license_period.additional_embedding_allowed:
+            return
+
+        # Check embedding quota
+        LicenseUsageServices._validate_embedding_quota(license_period, file_size_mb)
+
+    @staticmethod
+    def check_interaction_quota(tenant_id: int) -> None:
+        """
+        Check if a tenant can execute a specialist without exceeding interaction quota. As it is impossible to estimate
+        the number of interaction tokens, we only check if the interaction quota are exceeded. So we might have a
+        limited overrun.
+
+        Args:
+            tenant_id: ID of the tenant
+
+        Raises:
+            EveAIInteractionQuotaExceeded: If interaction quota would be exceeded
+            EveAINoActiveLicense: If no active license is found
+            EveAIException: For other errors
+        """
+        # Get active license period
+        license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
+        # Early return if both overruns are allowed - no need to check usage at all
+        if license_period.additional_interaction_allowed:
+            return
+
+        # Convert tokens to M tokens and check interaction quota
+        LicenseUsageServices._validate_interaction_quota(license_period)
+
+    @staticmethod
+    def _validate_storage_quota(license_period: LicensePeriod, additional_mb: float) -> None:
+        """Check storage quota and raise exception if exceeded"""
+        current_storage = license_period.license_usage.storage_mb_used or 0
+        projected_storage = current_storage + additional_mb
+        max_storage = license_period.max_storage_mb
+
+        # Hard limit check (we only get here if overruns are NOT allowed)
+        if projected_storage > max_storage:
+            raise EveAIStorageQuotaExceeded(
+                current_usage=current_storage,
+                limit=max_storage,
+                additional=additional_mb
+            )
+
+    @staticmethod
+    def _validate_embedding_quota(license_period: LicensePeriod, additional_mb: float) -> None:
+        """Check embedding quota and raise exception if exceeded"""
+        current_embedding = license_period.license_usage.embedding_mb_used or 0
+        projected_embedding = current_embedding + additional_mb
+        max_embedding = license_period.included_embedding_mb
+
+        # Hard limit check (we only get here if overruns are NOT allowed)
+        if projected_embedding > max_embedding:
+            raise EveAIEmbeddingQuotaExceeded(
+                current_usage=current_embedding,
+                limit=max_embedding,
+                additional=additional_mb
+            )
+
+    @staticmethod
+    def _validate_interaction_quota(license_period) -> None:
+        """Check interaction quota and raise exception if exceeded (tokens in millions). We might have an overrun!"""
+        current_tokens = license_period.license_usage.interaction_total_tokens_used / 1_000_000 or 0
+        max_tokens = license_period.included_interaction_tokens
+
+        # Hard limit check (we only get here if overruns are NOT allowed)
+        if current_tokens > max_tokens:
+            raise EveAIInteractionQuotaExceeded(
+                current_usage=current_tokens,
+                limit=max_tokens
+            )
+
+
+
+
+

common/services/interaction/asset_services.py

@@ -0,0 +1,9 @@
+from common.models.interaction import EveAIAsset
+from common.extensions import minio_client
+
+
+class AssetServices:
+
+    @staticmethod
+    def add_or_replace_asset_file(asset_id, file_data):
+        asset = EveAIAsset.query.get_or_404(asset_id)

common/services/interaction/capsule_services.py

@@ -0,0 +1,25 @@
+from datetime import datetime as dt, timezone as tz
+
+from common.models.interaction import EveAIDataCapsule
+from common.extensions import db
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+
+
+class CapsuleServices:
+    @staticmethod
+    def push_capsule_data(chat_session_id: str, type: str, type_version: str, configuration: dict, data: dict):
+        capsule = EveAIDataCapsule.query.filter_by(chat_session_id=chat_session_id, type=type, type_version=type_version).first()
+        if capsule:
+            # Update bestaande capsule als deze al bestaat
+            capsule.configuration = configuration
+            capsule.data = data
+            update_logging_information(capsule, dt.now(tz.utc))
+        else:
+            # Maak nieuwe capsule aan als deze nog niet bestaat
+            capsule = EveAIDataCapsule(chat_session_id=chat_session_id, type=type, type_version=type_version,
+                                       configuration=configuration, data=data)
+            set_logging_information(capsule, dt.now(tz.utc))
+            db.session.add(capsule)
+
+        db.session.commit()
+        return capsule

common/services/interaction/specialist_services.py

@@ -0,0 +1,237 @@
+import uuid
+from datetime import datetime as dt, timezone as tz
+from typing import Dict, Any, Tuple, Optional
+from flask import current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db, cache_manager
+from common.models.interaction import (
+    Specialist, EveAIAgent, EveAITask, EveAITool
+)
+from common.utils.celery_utils import current_celery
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+
+
+class SpecialistServices:
+    @staticmethod
+    def start_session() -> str:
+        return f"CHAT_SESSION_{uuid.uuid4()}"
+
+    @staticmethod
+    def execute_specialist(tenant_id, specialist_id, specialist_arguments, session_id, user_timezone) -> Dict[str, Any]:
+        task = current_celery.send_task(
+            'execute_specialist',
+            args=[tenant_id,
+                  specialist_id,
+                  specialist_arguments,
+                  session_id,
+                  user_timezone,
+                  ],
+            queue='llm_interactions'
+        )
+
+        return {
+            'task_id': task.id,
+            'status': 'queued',
+        }
+
+    @staticmethod
+    def initialize_specialist(specialist_id: int, specialist_type: str, specialist_version: str):
+        """
+        Initialize an agentic specialist by creating all its components based on configuration.
+
+        Args:
+            specialist_id: ID of the specialist to initialize
+            specialist_type: Type of the specialist
+            specialist_version: Version of the specialist type to use
+
+        Raises:
+            ValueError: If specialist not found or invalid configuration
+            SQLAlchemyError: If database operations fail
+        """
+        config = cache_manager.specialists_config_cache.get_config(specialist_type, specialist_version)
+        if not config:
+            raise ValueError(f"No configuration found for {specialist_type} version {specialist_version}")
+        if config['framework'] == 'langchain':
+            pass    # Langchain does not require additional items to be initialized. All configuration is in the specialist.
+
+        specialist = Specialist.query.get(specialist_id)
+        if not specialist:
+            raise ValueError(f"Specialist with ID {specialist_id} not found")
+
+        if config['framework'] == 'crewai':
+            SpecialistServices.initialize_crewai_specialist(specialist, config)
+
+    @staticmethod
+    def initialize_crewai_specialist(specialist: Specialist, config: Dict[str, Any]):
+        timestamp = dt.now(tz=tz.utc)
+
+        try:
+            # Initialize agents
+            if 'agents' in config:
+                for agent_config in config['agents']:
+                    SpecialistServices._create_agent(
+                        specialist_id=specialist.id,
+                        agent_type=agent_config['type'],
+                        agent_version=agent_config['version'],
+                        name=agent_config.get('name'),
+                        description=agent_config.get('description'),
+                        timestamp=timestamp
+                    )
+
+            # Initialize tasks
+            if 'tasks' in config:
+                for task_config in config['tasks']:
+                    SpecialistServices._create_task(
+                        specialist_id=specialist.id,
+                        task_type=task_config['type'],
+                        task_version=task_config['version'],
+                        name=task_config.get('name'),
+                        description=task_config.get('description'),
+                        timestamp=timestamp
+                    )
+
+            # Initialize tools
+            if 'tools' in config:
+                for tool_config in config['tools']:
+                    SpecialistServices._create_tool(
+                        specialist_id=specialist.id,
+                        tool_type=tool_config['type'],
+                        tool_version=tool_config['version'],
+                        name=tool_config.get('name'),
+                        description=tool_config.get('description'),
+                        timestamp=timestamp
+                    )
+
+            db.session.commit()
+            current_app.logger.info(f"Successfully initialized crewai specialist {specialist.id}")
+
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Database error initializing crewai specialist {specialist.id}: {str(e)}")
+            raise
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error initializing crewai specialist {specialist.id}: {str(e)}")
+            raise
+
+    @staticmethod
+    def _create_agent(
+            specialist_id: int,
+            agent_type: str,
+            agent_version: str,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            timestamp: Optional[dt] = None
+    ) -> EveAIAgent:
+        """Create an agent with the given configuration."""
+        if timestamp is None:
+            timestamp = dt.now(tz=tz.utc)
+
+        # Get agent configuration from cache
+        agent_config = cache_manager.agents_config_cache.get_config(agent_type, agent_version)
+
+        agent = EveAIAgent(
+            specialist_id=specialist_id,
+            name=name or agent_config.get('name', agent_type),
+            description=description or agent_config.get('metadata').get('description', ''),
+            type=agent_type,
+            type_version=agent_version,
+            role=None,
+            goal=None,
+            backstory=None,
+            tuning=False,
+            configuration=None,
+            arguments=None
+        )
+
+        set_logging_information(agent, timestamp)
+
+        db.session.add(agent)
+        current_app.logger.info(f"Created agent {agent.id} of type {agent_type}")
+        return agent
+
+    @staticmethod
+    def _create_task(
+            specialist_id: int,
+            task_type: str,
+            task_version: str,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            timestamp: Optional[dt] = None
+    ) -> EveAITask:
+        """Create a task with the given configuration."""
+        if timestamp is None:
+            timestamp = dt.now(tz=tz.utc)
+
+        # Get task configuration from cache
+        task_config = cache_manager.tasks_config_cache.get_config(task_type, task_version)
+
+        task = EveAITask(
+            specialist_id=specialist_id,
+            name=name or task_config.get('name', task_type),
+            description=description or task_config.get('metadata').get('description', ''),
+            type=task_type,
+            type_version=task_version,
+            task_description=None,
+            expected_output=None,
+            tuning=False,
+            configuration=None,
+            arguments=None,
+            context=None,
+            asynchronous=False,
+        )
+
+        set_logging_information(task, timestamp)
+
+        db.session.add(task)
+        current_app.logger.info(f"Created task {task.id} of type {task_type}")
+        return task
+
+    @staticmethod
+    def _create_tool(
+            specialist_id: int,
+            tool_type: str,
+            tool_version: str,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            timestamp: Optional[dt] = None
+    ) -> EveAITool:
+        """Create a tool with the given configuration."""
+        if timestamp is None:
+            timestamp = dt.now(tz=tz.utc)
+
+        # Get tool configuration from cache
+        tool_config = cache_manager.tools_config_cache.get_config(tool_type, tool_version)
+
+        tool = EveAITool(
+            specialist_id=specialist_id,
+            name=name or tool_config.get('name', tool_type),
+            description=description or tool_config.get('metadata').get('description', ''),
+            type=tool_type,
+            type_version=tool_version,
+            tuning=False,
+            configuration=None,
+            arguments=None,
+        )
+
+        set_logging_information(tool, timestamp)
+
+        db.session.add(tool)
+        current_app.logger.info(f"Created tool {tool.id} of type {tool_type}")
+        return tool
+
+    @staticmethod
+    def get_specialist_system_field(specialist_id, config_name, system_name):
+        """Get the value of a system field in a specialist's configuration. Returns the actual value, or None."""
+        specialist = Specialist.query.get(specialist_id)
+        if not specialist:
+            raise ValueError(f"Specialist with ID {specialist_id} not found")
+        config = cache_manager.specialists_config_cache.get_config(specialist.type, specialist.type_version)
+        if not config:
+            raise ValueError(f"No configuration found for {specialist.type} version {specialist.version}")
+        potential_field = config.get(config_name, None)
+        if potential_field:
+            if potential_field.type == 'system' and potential_field.system_name == system_name:
+                return specialist.configuration.get(config_name, None)
+        return None

common/services/user/__init__.py

@@ -0,0 +1,5 @@
+from common.services.user.user_services import UserServices
+from common.services.user.partner_services import PartnerServices
+from common.services.user.tenant_services import TenantServices
+
+__all__ = ['UserServices', 'PartnerServices', 'TenantServices']

common/services/user/partner_services.py

@@ -0,0 +1,53 @@
+from typing import List, Dict, Any
+
+from flask import session
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.models.entitlements import PartnerServiceLicenseTier
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService, EveAINoSessionPartner
+
+from common.utils.security_utils import current_user_has_role
+
+
+class PartnerServices:
+    @staticmethod
+    def get_allowed_license_tier_ids() -> List[int]:
+        """
+        Retrieve IDs of all License Tiers associated with the partner's management service
+
+        Returns:
+            List of license tier IDs
+
+        Raises:
+            EveAINoSessionPartner: If no partner is in the session
+            EveAINoManagementPartnerService: If partner has no management service
+        """
+        partner = session.get("partner", None)
+        if not partner:
+            raise EveAINoSessionPartner()
+
+        # Find a management service for this partner
+        management_service = next((service for service in session['partner']['services']
+                                   if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+        if not management_service:
+            raise EveAINoManagementPartnerService()
+        management_service_id = management_service['id']
+
+        # Query for all license tiers associated with this management service
+        associations = PartnerServiceLicenseTier.query.filter_by(
+            partner_service_id=management_service_id
+        ).all()
+
+        # Extract the license tier IDs
+        license_tier_ids = [assoc.license_tier_id for assoc in associations]
+
+        return license_tier_ids
+
+    @staticmethod
+    def get_management_service() -> Dict[str, Any]:
+        management_service = next((service for service in session['partner']['services']
+                                   if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+        return management_service
+
+
+

common/services/user/tenant_services.py

@@ -0,0 +1,175 @@
+from typing import Dict, List
+
+from flask import session, current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db, cache_manager
+from common.models.user import Partner, PartnerTenant, PartnerService, Tenant
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService
+from common.utils.model_logging_utils import set_logging_information
+from datetime import datetime as dt, timezone as tz
+
+from common.utils.security_utils import current_user_has_role
+
+
+class TenantServices:
+    @staticmethod
+    def associate_tenant_with_partner(tenant_id):
+        """Associate a tenant with a partner"""
+        try:
+            partner_id = session['partner']['id']
+            # Get partner service (MANAGEMENT_SERVICE type)
+            partner = Partner.query.get(partner_id)
+            if not partner:
+                return
+
+            # Find a management service for this partner
+            management_service = next((service for service in session['partner']['services']
+                                       if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+
+            if not management_service:
+                current_app.logger.error(f"No Management Service defined for partner {partner_id} "
+                                         f"while associating tenant {tenant_id} with partner.")
+                raise EveAINoManagementPartnerService()
+
+            # Create the association
+            tenant_partner = PartnerTenant(
+                partner_service_id=management_service['id'],
+                tenant_id=tenant_id,
+            )
+            set_logging_information(tenant_partner, dt.now(tz.utc))
+
+            db.session.add(tenant_partner)
+            db.session.commit()
+
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error associating tenant {tenant_id} with partner: {str(e)}")
+            raise e
+
+    @staticmethod
+    def get_available_types_for_tenant(tenant_id: int, config_type: str) -> Dict[str, Dict[str, str]]:
+        """
+        Get available configuration types for a tenant based on partner relationships
+
+        Args:
+            tenant_id: The tenant ID
+            config_type: The configuration type ('specialists', 'agents', 'tasks', etc.)
+
+        Returns:
+            Dictionary of available types for the tenant
+        """
+        # Get the appropriate cache handler based on config_type
+        cache_handler = None
+        if config_type == 'specialists':
+            cache_handler = cache_manager.specialists_types_cache
+        elif config_type == 'agents':
+            cache_handler = cache_manager.agents_types_cache
+        elif config_type == 'tasks':
+            cache_handler = cache_manager.tasks_types_cache
+        elif config_type == 'tools':
+            cache_handler = cache_manager.tools_types_cache
+        elif config_type == 'catalogs':
+            cache_handler = cache_manager.catalogs_types_cache
+        elif config_type == 'retrievers':
+            cache_handler = cache_manager.retrievers_types_cache
+        else:
+            raise ValueError(f"Unsupported config type: {config_type}")
+
+        # Get all types with their metadata (including partner info)
+        all_types = cache_handler.get_types()
+
+        # Filter to include:
+        # 1. Types with no partner (global)
+        # 2. Types with partners that have a SPECIALIST_SERVICE relationship with this tenant
+        available_partners = TenantServices.get_tenant_partner_specialist_denominators(tenant_id)
+
+        available_types = {
+            type_id: info for type_id, info in all_types.items()
+            if info.get('partner') is None or info.get('partner') in available_partners
+        }
+
+        return available_types
+
+    @staticmethod
+    def get_tenant_partner_specialist_denominators(tenant_id: int) -> List[str]:
+        """
+        Get names of partners that have a SPECIALIST_SERVICE relationship with this tenant, that can be used for
+        filtering configurations.
+
+        Args:
+            tenant_id: The tenant ID
+
+        Returns:
+            List of partner names (tenant names)
+        """
+        # Find all PartnerTenant relationships for this tenant
+        partner_service_denominators = []
+        try:
+            # Get all partner services of type SPECIALIST_SERVICE
+            specialist_services = (
+                PartnerService.query
+                .filter_by(type='SPECIALIST_SERVICE')
+                .all()
+            )
+
+            if not specialist_services:
+                return []
+
+            # Find tenant relationships with these services
+            partner_tenants = (
+                PartnerTenant.query
+                .filter_by(tenant_id=tenant_id)
+                .filter(PartnerTenant.partner_service_id.in_([svc.id for svc in specialist_services]))
+                .all()
+            )
+
+            # Get the partner names (their tenant names)
+            for pt in partner_tenants:
+                partner_service = (
+                    PartnerService.query
+                    .filter_by(id=pt.partner_service_id)
+                    .first()
+                )
+
+                if partner_service:
+                    partner_service_denominators.append(partner_service.configuration.get("specialist_denominator", ""))
+
+        except SQLAlchemyError as e:
+            current_app.logger.error(f"Database error retrieving partner names: {str(e)}")
+
+        return partner_service_denominators
+
+    @staticmethod
+    def can_use_specialist_type(tenant_id: int, specialist_type: str) -> bool:
+            """
+            Check if a tenant can use a specific specialist type
+
+            Args:
+                tenant_id: The tenant ID
+                specialist_type: The specialist type ID
+
+            Returns:
+                True if the tenant can use the specialist type, False otherwise
+            """
+            # Get the specialist type definition
+            try:
+                specialist_types = cache_manager.specialists_types_cache.get_types()
+                specialist_def = specialist_types.get(specialist_type)
+
+                if not specialist_def:
+                    return False
+
+                # If it's a global specialist, anyone can use it
+                if specialist_def.get('partner') is None:
+                    return True
+
+                # If it's a partner-specific specialist, check if tenant has access
+                partner_name = specialist_def.get('partner')
+                available_partners = TenantServices.get_tenant_partner_specialist_denominators(tenant_id)
+
+                return partner_name in available_partners
+
+            except Exception as e:
+                current_app.logger.error(f"Error checking specialist type access: {str(e)}")
+                return False

common/services/user/user_services.py

@@ -0,0 +1,95 @@
+from flask import session
+
+from common.models.user import Partner, Role, PartnerTenant
+
+from common.utils.eveai_exceptions import EveAIRoleAssignmentException
+from common.utils.security_utils import current_user_has_role
+
+
+class UserServices:
+    @staticmethod
+    def get_assignable_roles():
+        """Retrieves roles that can be assigned to a user depending on the current user logged in,
+        and the active tenant for the session"""
+        current_tenant_id = session.get('tenant').get('id', None)
+        effective_role_names = []
+        if current_tenant_id == 1:
+            if current_user_has_role("Super User"):
+                effective_role_names.append("Super User")
+        elif current_tenant_id:
+            if current_user_has_role("Tenant Admin"):
+                effective_role_names.append("Tenant Admin")
+            if current_user_has_role("Partner Admin") or current_user_has_role("Super User"):
+                effective_role_names.append("Tenant Admin")
+                if session.get('partner'):
+                    if session.get('partner').get('tenant_id') == current_tenant_id:
+                        effective_role_names.append("Partner Admin")
+        effective_role_names = list(set(effective_role_names))
+        effective_roles = [(role.id, role.name) for role in
+                           Role.query.filter(Role.name.in_(effective_role_names)).all()]
+        return effective_roles
+
+    @staticmethod
+    def validate_role_assignments(role_ids):
+        """Validate a set of role assignments, raising exception for first invalid role"""
+        assignable_roles = UserServices.get_assignable_roles()
+        assignable_role_ids = {role[0] for role in assignable_roles}
+        role_id_set = set(role_ids)
+        return role_id_set.issubset(assignable_role_ids)
+
+    @staticmethod
+    def can_user_edit_tenant(tenant_id) -> bool:
+        if current_user_has_role('Super User'):
+            return True
+        elif current_user_has_role('Partner Admin'):
+            partner = session.get('partner', None)
+            if partner and partner["tenant_id"] == tenant_id:
+                return True
+            partner_service = next((service for service in session['partner']['services']
+                                    if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+            if not partner_service:
+                return False
+            else:
+                partner_tenant = PartnerTenant.query.filter(
+                    PartnerTenant.tenant_id == tenant_id,
+                    PartnerTenant.partner_service_id == partner_service['id'],
+                ).first()
+                if partner_tenant:
+                    return True
+                else:
+                    return False
+        else:
+            return False
+
+    @staticmethod
+    def can_user_create_tenant() -> bool:
+        if current_user_has_role('Super User'):
+            return True
+        elif current_user_has_role('Partner Admin'):
+            partner_id = session['partner']['id']
+            partner_service = next((service for service in session['partner']['services']
+                                    if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+            if not partner_service:
+                return False
+            else:
+                partner_permissions = partner_service.get('permissions', None)
+                return partner_permissions.get('can_create_tenant', False)
+        else:
+            return False
+
+    @staticmethod
+    def can_user_assign_license() -> bool:
+        if current_user_has_role('Super User'):
+            return True
+        elif current_user_has_role('Partner Admin'):
+            partner_id = session['partner']['id']
+            partner_service = next((service for service in session['partner']['services']
+                                    if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+            if not partner_service:
+                return False
+            else:
+                partner_permissions = partner_service.get('permissions', None)
+                return partner_permissions.get('can_assign_license', False)
+        else:
+            return False
+

common/services/utils/human_answer_services.py

@@ -0,0 +1,108 @@
+from flask import current_app, session
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+
+from common.utils.business_event import BusinessEvent
+from common.utils.business_event_context import current_event
+from common.utils.model_utils import get_template
+from eveai_chat_workers.outputs.globals.a2q_output.q_a_output_v1_0 import A2QOutput
+from eveai_chat_workers.outputs.globals.q_a_output.q_a_output_v1_0 import QAOutput
+
+
+class HumanAnswerServices:
+    @staticmethod
+    def check_affirmative_answer(tenant_id: int, question: str, answer: str, language_iso: str) -> bool:
+        return HumanAnswerServices._check_answer(tenant_id, question, answer, language_iso, "check_affirmative_answer",
+                                                 "Check Affirmative Answer")
+
+    @staticmethod
+    def check_additional_information(tenant_id: int, question: str, answer: str, language_iso: str) -> bool:
+        result = HumanAnswerServices._check_answer(tenant_id, question, answer, language_iso,
+                                                   "check_additional_information", "Check Additional Information")
+
+        return result
+
+    @staticmethod
+    def get_answer_to_question(tenant_id: int, question: str, answer: str, language_iso: str) -> str:
+
+        language = HumanAnswerServices._process_arguments(question, answer, language_iso)
+        span_name = "Get Answer To Question"
+        template_name = "get_answer_to_question"
+
+        if not current_event:
+            with BusinessEvent('Answer Check Service', tenant_id):
+                with current_event.create_span(span_name):
+                    return HumanAnswerServices._get_answer_to_question_logic(question, answer, language, template_name)
+        else:
+            with current_event.create_span('Check Affirmative Answer'):
+                return HumanAnswerServices._get_answer_to_question_logic(question, answer, language, template_name)
+
+    @staticmethod
+    def _check_answer(tenant_id: int, question: str, answer: str, language_iso: str, template_name: str,
+                      span_name: str) -> bool:
+        language = HumanAnswerServices._process_arguments(question, answer, language_iso)
+        if not current_event:
+            with BusinessEvent('Answer Check Service', tenant_id):
+                with current_event.create_span(span_name):
+                    return HumanAnswerServices._check_answer_logic(question, answer, language, template_name)
+        else:
+            with current_event.create_span(span_name):
+                return HumanAnswerServices._check_answer_logic(question, answer, language, template_name)
+
+    @staticmethod
+    def _check_answer_logic(question: str, answer: str, language: str, template_name: str) -> bool:
+        prompt_params = {
+            'question': question,
+            'answer': answer,
+            'language': language,
+        }
+
+        template, llm = get_template(template_name)
+        check_answer_prompt = ChatPromptTemplate.from_template(template)
+        setup = RunnablePassthrough()
+
+        output_schema = QAOutput
+        structured_llm = llm.with_structured_output(output_schema)
+
+        chain = (setup | check_answer_prompt | structured_llm )
+
+        raw_answer = chain.invoke(prompt_params)
+
+        return raw_answer.answer
+
+    @staticmethod
+    def _get_answer_to_question_logic(question: str, answer: str, language: str, template_name: str) \
+            -> str:
+        prompt_params = {
+            'question': question,
+            'answer': answer,
+            'language': language,
+        }
+
+        template, llm = get_template(template_name)
+        check_answer_prompt = ChatPromptTemplate.from_template(template)
+        setup = RunnablePassthrough()
+
+        output_schema = A2QOutput
+        structured_llm = llm.with_structured_output(output_schema)
+
+        chain = (setup | check_answer_prompt | structured_llm)
+
+        raw_answer = chain.invoke(prompt_params)
+
+        return raw_answer.answer
+
+    @staticmethod
+    def _process_arguments(question, answer, language_iso: str) -> str:
+        if language_iso.strip() == '':
+            raise ValueError("Language cannot be empty")
+        language = current_app.config.get('SUPPORTED_LANGUAGE_ISO639_1_LOOKUP').get(language_iso)
+        if language is None:
+            raise ValueError(f"Unsupported language: {language_iso}")
+        if question.strip() == '':
+            raise ValueError("Question cannot be empty")
+        if answer.strip() == '':
+            raise ValueError("Answer cannot be empty")
+
+        return language

common/services/utils/translation_services.py

@@ -0,0 +1,148 @@
+import json
+from typing import Dict, Any, Optional
+
+from flask import session
+
+from common.extensions import cache_manager
+from common.utils.business_event import BusinessEvent
+from common.utils.business_event_context import current_event
+
+class TranslationServices:
+
+    @staticmethod
+    def translate_config(tenant_id: int, config_data: Dict[str, Any], field_config: str, target_language: str,
+                         source_language: Optional[str] = None, context: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Vertaalt een configuratie op basis van een veld-configuratie.
+
+        Args:
+            tenant_id: Identificatie van de tenant waarvoor we de vertaling doen.
+            config_data: Een dictionary of JSON (die dan wordt geconverteerd naar een dictionary) met configuratiegegevens
+            field_config: De naam van een veld-configuratie (bijv. 'fields')
+            target_language: De taal waarnaar vertaald moet worden
+            source_language: Optioneel, de brontaal van de configuratie
+            context: Optioneel, een specifieke context voor de vertaling
+
+        Returns:
+            Een dictionary met de vertaalde configuratie
+        """
+        config_type = config_data.get('type', 'Unknown')
+        config_version = config_data.get('version', 'Unknown')
+        span_name = f"{config_type}-{config_version}-{field_config}"
+
+        if current_event:
+            with current_event.create_span(span_name):
+                translated_config = TranslationServices._translate_config(tenant_id, config_data, field_config,
+                                                                          target_language, source_language, context)
+                return translated_config
+        else:
+            with BusinessEvent('Config Translation Service', tenant_id):
+                with current_event.create_span(span_name):
+                    translated_config = TranslationServices._translate_config(tenant_id, config_data, field_config,
+                                                                              target_language, source_language, context)
+                    return translated_config
+
+    @staticmethod
+    def _translate_config(tenant_id: int, config_data: Dict[str, Any], field_config: str, target_language: str,
+                          source_language: Optional[str] = None, context: Optional[str] = None) -> Dict[str, Any]:
+
+        # Zorg ervoor dat we een dictionary hebben
+        if isinstance(config_data, str):
+            config_data = json.loads(config_data)
+
+        # Maak een kopie van de originele data om te wijzigen
+        translated_config = config_data.copy()
+
+        # Haal type en versie op voor de Business Event span
+        config_type = config_data.get('type', 'Unknown')
+        config_version = config_data.get('version', 'Unknown')
+
+        if field_config in config_data:
+            fields = config_data[field_config]
+
+            # Haal description uit metadata voor context als geen context is opgegeven
+            description_context = ""
+            if not context and 'metadata' in config_data and 'description' in config_data['metadata']:
+                description_context = config_data['metadata']['description']
+
+            # Loop door elk veld in de configuratie
+            for field_name, field_data in fields.items():
+                # Vertaal name als het bestaat en niet leeg is
+                if 'name' in field_data and field_data['name']:
+                    # Gebruik context indien opgegeven, anders description_context
+                    field_context = context if context else description_context
+                    translated_name = cache_manager.translation_cache.get_translation(
+                        text=field_data['name'],
+                        target_lang=target_language,
+                        source_lang=source_language,
+                        context=field_context
+                    )
+                    if translated_name:
+                        translated_config[field_config][field_name]['name'] = translated_name.translated_text
+
+                if 'title' in field_data and field_data['title']:
+                    # Gebruik context indien opgegeven, anders description_context
+                    field_context = context if context else description_context
+                    translated_title = cache_manager.translation_cache.get_translation(
+                        text=field_data['title'],
+                        target_lang=target_language,
+                        source_lang=source_language,
+                        context=field_context
+                    )
+                    if translated_title:
+                        translated_config[field_config][field_name]['title'] = translated_title.translated_text
+
+                # Vertaal description als het bestaat en niet leeg is
+                if 'description' in field_data and field_data['description']:
+                    # Gebruik context indien opgegeven, anders description_context
+                    field_context = context if context else description_context
+                    translated_desc = cache_manager.translation_cache.get_translation(
+                        text=field_data['description'],
+                        target_lang=target_language,
+                        source_lang=source_language,
+                        context=field_context
+                    )
+                    if translated_desc:
+                        translated_config[field_config][field_name]['description'] = translated_desc.translated_text
+
+                # Vertaal context als het bestaat en niet leeg is
+                if 'context' in field_data and field_data['context']:
+                    translated_ctx = cache_manager.translation_cache.get_translation(
+                        text=field_data['context'],
+                        target_lang=target_language,
+                        source_lang=source_language,
+                        context=context
+                    )
+                    if translated_ctx:
+                        translated_config[field_config][field_name]['context'] = translated_ctx.translated_text
+
+                # vertaal allowed values als het veld bestaat en de waarden niet leeg zijn.
+                if 'allowed_values' in field_data and field_data['allowed_values']:
+                    translated_allowed_values = []
+                    for allowed_value in field_data['allowed_values']:
+                        translated_allowed_value = cache_manager.translation_cache.get_translation(
+                            text=allowed_value,
+                            target_lang=target_language,
+                            source_lang=source_language,
+                            context=context
+                        )
+                        translated_allowed_values.append(translated_allowed_value.translated_text)
+                    if translated_allowed_values:
+                        translated_config[field_config][field_name]['allowed_values'] = translated_allowed_values
+
+            return translated_config
+
+    @staticmethod
+    def translate(tenant_id: int, text: str, target_language: str, source_language: Optional[str] = None,
+                  context: Optional[str] = None)-> str:
+        if current_event:
+            with current_event.create_span('Translation'):
+                translation_cache = cache_manager.translation_cache.get_translation(text, target_language,
+                                                                                    source_language, context)
+                return translation_cache.translated_text
+        else:
+            with BusinessEvent('Translation Service', tenant_id):
+                with current_event.create_span('Translation'):
+                    translation_cache = cache_manager.translation_cache.get_translation(text, target_language,
+                                                                                        source_language, context)
+                    return translation_cache.translated_text

common/utils/asset_utils.py

@@ -4,59 +4,9 @@ from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
 
 from common.extensions import cache_manager, minio_client, db
-from common.models.interaction import EveAIAsset, EveAIAssetVersion
-from common.utils.document_utils import mark_tenant_storage_dirty
+from common.models.interaction import EveAIAsset
 from common.utils.model_logging_utils import set_logging_information
 
 
-def create_asset_stack(api_input, tenant_id):
-    type_version = cache_manager.assets_version_tree_cache.get_latest_version(api_input['type'])
-    api_input['type_version'] = type_version
-    new_asset = create_asset(api_input, tenant_id)
-    new_asset_version = create_version_for_asset(new_asset, tenant_id)
-    db.session.add(new_asset)
-    db.session.add(new_asset_version)
-
-    try:
-        db.session.commit()
-    except SQLAlchemyError as e:
-        current_app.logger.error(f"Could not add asset for tenant {tenant_id}: {str(e)}")
-        db.session.rollback()
-        raise e
-
-    return new_asset, new_asset_version
-
-
-def create_asset(api_input, tenant_id):
-    new_asset = EveAIAsset()
-    new_asset.name = api_input['name']
-    new_asset.description = api_input['description']
-    new_asset.type = api_input['type']
-    new_asset.type_version = api_input['type_version']
-    if api_input['valid_from'] and api_input['valid_from'] != '':
-        new_asset.valid_from = api_input['valid_from']
-    else:
-        new_asset.valid_from = dt.now(tz.utc)
-    new_asset.valid_to = api_input['valid_to']
-    set_logging_information(new_asset, dt.now(tz.utc))
-
-    return new_asset
-
-
-def create_version_for_asset(asset, tenant_id):
-    new_asset_version = EveAIAssetVersion()
-    new_asset_version.asset = asset
-    new_asset_version.bucket_name = minio_client.create_tenant_bucket(tenant_id)
-    set_logging_information(new_asset_version, dt.now(tz.utc))
-
-    return new_asset_version
-
-
-def add_asset_version_file(asset_version, field_name, file, tenant_id):
-    object_name, file_size = minio_client.upload_file(asset_version.bucket_name, asset_version.id, field_name,
-                                                      file.content_type)
-    mark_tenant_storage_dirty(tenant_id)
-    return object_name
-
 
 

common/utils/business_event.py

@@ -6,16 +6,18 @@ from datetime import datetime
 from typing import Dict, Any, Optional, List
 from datetime import datetime as dt, timezone as tz
 import logging
-from prometheus_client import Counter, Histogram, Gauge, Summary
+
+from flask import current_app
+from prometheus_client import Counter, Histogram, Gauge, Summary, push_to_gateway, REGISTRY
 
 from .business_event_context import BusinessEventContext
 from common.models.entitlements import BusinessEventLog
 from common.extensions import db
 from .celery_utils import current_celery
-from common.utils.performance_monitoring import EveAIMetrics
+from common.utils.prometheus_utils import sanitize_label
 
 # Standard duration buckets for all histograms
-DURATION_BUCKETS = EveAIMetrics.get_standard_buckets()
+DURATION_BUCKETS = [0.1, 0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 240, 360, float('inf')]
 
 # Prometheus metrics for business events
 TRACE_COUNTER = Counter(
@@ -104,6 +106,7 @@ class BusinessEvent:
             'total_tokens': 0,
             'prompt_tokens': 0,
             'completion_tokens': 0,
+            'nr_of_pages': 0,
             'total_time': 0,
             'call_count': 0,
             'interaction_type': None
@@ -112,14 +115,17 @@ class BusinessEvent:
 
         # Prometheus label values must be strings
         self.tenant_id_str = str(self.tenant_id)
+        self.event_type_str = sanitize_label(self.event_type)
         self.specialist_id_str = str(self.specialist_id) if self.specialist_id else ""
         self.specialist_type_str = str(self.specialist_type) if self.specialist_type else ""
-        self.specialist_type_version_str = str(self.specialist_type_version) if self.specialist_type_version else ""
+        self.specialist_type_version_str = sanitize_label(str(self.specialist_type_version)) \
+            if self.specialist_type_version else ""
+        self.span_name_str = ""
 
         # Increment concurrent events gauge when initialized
         CONCURRENT_TRACES.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
+            event_type=self.event_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
@@ -128,12 +134,14 @@ class BusinessEvent:
         # Increment trace counter
         TRACE_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
+            event_type=self.event_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
         ).inc()
 
+        self._push_to_gateway()
+
     def update_attribute(self, attribute: str, value: any):
         if hasattr(self, attribute):
             setattr(self, attribute, value)
@@ -143,78 +151,86 @@ class BusinessEvent:
             elif attribute == 'specialist_type':
                 self.specialist_type_str = str(value) if value else ""
             elif attribute == 'specialist_type_version':
-                self.specialist_type_version_str = str(value) if value else ""
+                self.specialist_type_version_str = sanitize_label(str(value)) if value else ""
             elif attribute == 'tenant_id':
                 self.tenant_id_str = str(value)
+            elif attribute == 'event_type':
+                self.event_type_str = sanitize_label(value)
+            elif attribute == 'span_name':
+                self.span_name_str = sanitize_label(value)
         else:
             raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'")
 
     def update_llm_metrics(self, metrics: dict):
-        self.llm_metrics['total_tokens'] += metrics['total_tokens']
-        self.llm_metrics['prompt_tokens'] += metrics['prompt_tokens']
-        self.llm_metrics['completion_tokens'] += metrics['completion_tokens']
-        self.llm_metrics['total_time'] += metrics['time_elapsed']
+        self.llm_metrics['total_tokens'] += metrics.get('total_tokens', 0)
+        self.llm_metrics['prompt_tokens'] += metrics.get('prompt_tokens', 0)
+        self.llm_metrics['completion_tokens'] += metrics.get('completion_tokens', 0)
+        self.llm_metrics['nr_of_pages'] += metrics.get('nr_of_pages', 0)
+        self.llm_metrics['total_time'] += metrics.get('time_elapsed', 0)
         self.llm_metrics['call_count'] += 1
         self.llm_metrics['interaction_type'] = metrics['interaction_type']
 
         # Track in Prometheus metrics
-        interaction_type = metrics['interaction_type']
+        interaction_type_str = sanitize_label(metrics['interaction_type']) if metrics['interaction_type'] else ""
 
         # Track token usage
         LLM_TOKENS_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            interaction_type=interaction_type,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
             token_type='total',
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
-        ).inc(metrics['total_tokens'])
+        ).inc(metrics.get('total_tokens', 0))
 
         LLM_TOKENS_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            interaction_type=interaction_type,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
             token_type='prompt',
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
-        ).inc(metrics['prompt_tokens'])
+        ).inc(metrics.get('prompt_tokens', 0))
 
         LLM_TOKENS_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            interaction_type=interaction_type,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
             token_type='completion',
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
-        ).inc(metrics['completion_tokens'])
+        ).inc(metrics.get('completion_tokens', 0))
 
         # Track duration
         LLM_DURATION.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            interaction_type=interaction_type,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
-        ).observe(metrics['time_elapsed'])
+        ).observe(metrics.get('time_elapsed', 0))
 
         # Track call count
         LLM_CALLS_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            interaction_type=interaction_type,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
         ).inc()
 
+        self._push_to_gateway()
+
     def reset_llm_metrics(self):
         self.llm_metrics['total_tokens'] = 0
         self.llm_metrics['prompt_tokens'] = 0
         self.llm_metrics['completion_tokens'] = 0
+        self.llm_metrics['nr_of_pages'] = 0
         self.llm_metrics['total_time'] = 0
         self.llm_metrics['call_count'] = 0
         self.llm_metrics['interaction_type'] = None
@@ -236,6 +252,7 @@ class BusinessEvent:
         # Set the new span info
         self.span_id = new_span_id
         self.span_name = span_name
+        self.span_name_str = sanitize_label(span_name) if span_name else ""
         self.parent_span_id = parent_span_id
 
         # Track start time for the span
@@ -244,8 +261,8 @@ class BusinessEvent:
         # Increment span metrics - using span_name as activity_name for metrics
         SPAN_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            activity_name=span_name,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
@@ -254,13 +271,15 @@ class BusinessEvent:
         # Increment concurrent spans gauge
         CONCURRENT_SPANS.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            activity_name=span_name,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
         ).inc()
 
+        self._push_to_gateway()
+
         self.log(f"Start")
 
         try:
@@ -272,8 +291,8 @@ class BusinessEvent:
             # Observe span duration
             SPAN_DURATION.labels(
                 tenant_id=self.tenant_id_str,
-                event_type=self.event_type,
-                activity_name=span_name,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
                 specialist_id=self.specialist_id_str,
                 specialist_type=self.specialist_type_str,
                 specialist_type_version=self.specialist_type_version_str
@@ -282,13 +301,15 @@ class BusinessEvent:
             # Decrement concurrent spans gauge
             CONCURRENT_SPANS.labels(
                 tenant_id=self.tenant_id_str,
-                event_type=self.event_type,
-                activity_name=span_name,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
                 specialist_id=self.specialist_id_str,
                 specialist_type=self.specialist_type_str,
                 specialist_type_version=self.specialist_type_version_str
             ).dec()
 
+            self._push_to_gateway()
+
             if self.llm_metrics['call_count'] > 0:
                 self.log_final_metrics()
                 self.reset_llm_metrics()
@@ -296,10 +317,12 @@ class BusinessEvent:
             # Restore the previous span info
             if self.spans:
                 self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+                self.span_name_str = sanitize_label(span_name) if span_name else ""
             else:
                 self.span_id = None
                 self.span_name = None
                 self.parent_span_id = None
+                self.span_name_str = ""
 
     @asynccontextmanager
     async def create_span_async(self, span_name: str):
@@ -314,6 +337,7 @@ class BusinessEvent:
         # Set the new span info
         self.span_id = new_span_id
         self.span_name = span_name
+        self.span_name_str = sanitize_label(span_name) if span_name else ""
         self.parent_span_id = parent_span_id
 
         # Track start time for the span
@@ -322,8 +346,8 @@ class BusinessEvent:
         # Increment span metrics - using span_name as activity_name for metrics
         SPAN_COUNTER.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            activity_name=span_name,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
@@ -332,13 +356,15 @@ class BusinessEvent:
         # Increment concurrent spans gauge
         CONCURRENT_SPANS.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
-            activity_name=span_name,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
         ).inc()
 
+        self._push_to_gateway()
+
         self.log(f"Start")
 
         try:
@@ -350,8 +376,8 @@ class BusinessEvent:
             # Observe span duration
             SPAN_DURATION.labels(
                 tenant_id=self.tenant_id_str,
-                event_type=self.event_type,
-                activity_name=span_name,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
                 specialist_id=self.specialist_id_str,
                 specialist_type=self.specialist_type_str,
                 specialist_type_version=self.specialist_type_version_str
@@ -360,13 +386,15 @@ class BusinessEvent:
             # Decrement concurrent spans gauge
             CONCURRENT_SPANS.labels(
                 tenant_id=self.tenant_id_str,
-                event_type=self.event_type,
-                activity_name=span_name,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
                 specialist_id=self.specialist_id_str,
                 specialist_type=self.specialist_type_str,
                 specialist_type_version=self.specialist_type_version_str
             ).dec()
 
+            self._push_to_gateway()
+
             if self.llm_metrics['call_count'] > 0:
                 self.log_final_metrics()
                 self.reset_llm_metrics()
@@ -374,10 +402,12 @@ class BusinessEvent:
             # Restore the previous span info
             if self.spans:
                 self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+                self.span_name_str = sanitize_label(span_name) if span_name else ""
             else:
                 self.span_id = None
                 self.span_name = None
                 self.parent_span_id = None
+                self.span_name_str = ""
 
     def log(self, message: str, level: str = 'info', extra_fields: Dict[str, Any] = None):
         log_data = {
@@ -429,10 +459,11 @@ class BusinessEvent:
             'specialist_type': self.specialist_type,
             'specialist_type_version': self.specialist_type_version,
             'environment': self.environment,
-            'llm_metrics_total_tokens': metrics['total_tokens'],
-            'llm_metrics_prompt_tokens': metrics['prompt_tokens'],
-            'llm_metrics_completion_tokens': metrics['completion_tokens'],
-            'llm_metrics_total_time': metrics['time_elapsed'],
+            'llm_metrics_total_tokens': metrics.get('total_tokens', 0),
+            'llm_metrics_prompt_tokens': metrics.get('prompt_tokens', 0),
+            'llm_metrics_completion_tokens': metrics.get('completion_tokens', 0),
+            'llm_metrics_nr_of_pages': metrics.get('nr_of_pages', 0),
+            'llm_metrics_total_time': metrics.get('time_elapsed', 0),
             'llm_interaction_type': metrics['interaction_type'],
             'message': message,
         }
@@ -460,6 +491,7 @@ class BusinessEvent:
             'llm_metrics_total_tokens': self.llm_metrics['total_tokens'],
             'llm_metrics_prompt_tokens': self.llm_metrics['prompt_tokens'],
             'llm_metrics_completion_tokens': self.llm_metrics['completion_tokens'],
+            'llm_metrics_nr_of_pages': self.llm_metrics['nr_of_pages'],
             'llm_metrics_total_time': self.llm_metrics['total_time'],
             'llm_metrics_call_count': self.llm_metrics['call_count'],
             'llm_interaction_type': self.llm_metrics['interaction_type'],
@@ -526,6 +558,17 @@ class BusinessEvent:
             # Clear the buffer after sending
             self._log_buffer = []
 
+    def _push_to_gateway(self):
+        # Push metrics to the gateway
+        try:
+            push_to_gateway(
+                current_app.config['PUSH_GATEWAY_URL'],
+                job=current_app.config['COMPONENT_NAME'],
+                registry=REGISTRY
+            )
+        except Exception as e:
+            current_app.logger.error(f"Failed to push metrics to Prometheus Push Gateway: {e}")
+
     def __enter__(self):
         self.trace_start_time = time.time()
         self.log(f'Starting Trace for {self.event_type}')
@@ -537,7 +580,7 @@ class BusinessEvent:
         # Record trace duration
         TRACE_DURATION.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
+            event_type=self.event_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
@@ -546,18 +589,22 @@ class BusinessEvent:
         # Decrement concurrent traces gauge
         CONCURRENT_TRACES.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
+            event_type=self.event_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
         ).dec()
 
+        self._push_to_gateway()
+
         if self.llm_metrics['call_count'] > 0:
             self.log_final_metrics()
             self.reset_llm_metrics()
 
         self.log(f'Ending Trace for {self.event_type}', extra_fields={'trace_duration': trace_total_time})
         self._flush_log_buffer()
+
+
         return BusinessEventContext(self).__exit__(exc_type, exc_val, exc_tb)
 
     async def __aenter__(self):
@@ -571,7 +618,7 @@ class BusinessEvent:
         # Record trace duration
         TRACE_DURATION.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
+            event_type=self.event_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
@@ -580,12 +627,14 @@ class BusinessEvent:
         # Decrement concurrent traces gauge
         CONCURRENT_TRACES.labels(
             tenant_id=self.tenant_id_str,
-            event_type=self.event_type,
+            event_type=self.event_type_str,
             specialist_id=self.specialist_id_str,
             specialist_type=self.specialist_type_str,
             specialist_type_version=self.specialist_type_version_str
         ).dec()
 
+        self._push_to_gateway()
+
         if self.llm_metrics['call_count'] > 0:
             self.log_final_metrics()
             self.reset_llm_metrics()

common/utils/cache/config_cache.py

@@ -7,7 +7,7 @@ from flask import current_app
 
 from common.utils.cache.base import CacheHandler, CacheKey
 from config.type_defs import agent_types, task_types, tool_types, specialist_types, retriever_types, prompt_types, \
-    catalog_types
+    catalog_types, partner_service_types, processor_types, customisation_types, specialist_form_types, capsule_types
 
 
 def is_major_minor(version: str) -> bool:
@@ -59,16 +59,10 @@ class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
         """Set the version tree cache dependency."""
         self.version_tree_cache = cache
 
-    def _load_specific_config(self, type_name: str, version_str: str) -> Dict[str, Any]:
+    def _load_specific_config(self, type_name: str, version_str: str = 'latest') -> Dict[str, Any]:
         """
         Load a specific configuration version
-
-        Args:
-            type_name: Type name
-            version_str: Version string
-
-        Returns:
-            Configuration data
+        Automatically handles global vs partner-specific configs
         """
         version_tree = self.version_tree_cache.get_versions(type_name)
         versions = version_tree['versions']
@@ -79,11 +73,16 @@ class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
         if version_str not in versions:
             raise ValueError(f"Version {version_str} not found for {type_name}")
 
-        file_path = versions[version_str]['file_path']
+        version_info = versions[version_str]
+        file_path = version_info['file_path']
+        partner = version_info.get('partner')
 
         try:
             with open(file_path) as f:
                 config = yaml.safe_load(f)
+                # Add partner information to the config
+                if partner:
+                    config['partner'] = partner
                 return config
         except Exception as e:
             raise ValueError(f"Error loading config from {file_path}: {e}")
@@ -133,20 +132,37 @@ class BaseConfigVersionTreeCacheHandler(CacheHandler[Dict[str, Any]]):
     def _load_version_tree(self, type_name: str) -> Dict[str, Any]:
         """
         Load version tree for a specific type without loading full configurations
-
-        Args:
-            type_name: Name of configuration type
-
-        Returns:
-            Dict containing available versions and their metadata
+        Checks both global and partner-specific directories
         """
-        type_path = Path(self._config_dir) / type_name
-        if not type_path.exists():
+        # First check the global path
+        global_path = Path(self._config_dir) / "globals" / type_name
+
+        # If global path doesn't exist, check if the type exists directly in the root
+        # (for backward compatibility)
+        if not global_path.exists():
+            global_path = Path(self._config_dir) / type_name
+
+        if not global_path.exists():
+            # Check if it exists in any partner subdirectories
+            partner_dirs = [d for d in Path(self._config_dir).iterdir()
+                            if d.is_dir() and d.name != "globals"]
+
+            for partner_dir in partner_dirs:
+                partner_type_path = partner_dir / type_name
+                if partner_type_path.exists():
+                    # Found in partner directory
+                    return self._load_versions_from_path(partner_type_path)
+
+            # If we get here, the type wasn't found anywhere
             raise ValueError(f"No configuration found for type {type_name}")
 
-        version_files = list(type_path.glob('*.yaml'))
+        return self._load_versions_from_path(global_path)
+
+    def _load_versions_from_path(self, path: Path) -> Dict[str, Any]:
+        """Load all versions from a specific path"""
+        version_files = list(path.glob('*.yaml'))
         if not version_files:
-            raise ValueError(f"No versions found for type {type_name}")
+            raise ValueError(f"No versions found in {path}")
 
         versions = {}
         latest_version = None
@@ -160,9 +176,17 @@ class BaseConfigVersionTreeCacheHandler(CacheHandler[Dict[str, Any]]):
                 with open(file_path) as f:
                     yaml_data = yaml.safe_load(f)
                     metadata = yaml_data.get('metadata', {})
+                    # Add partner information if available
+                    partner = None
+                    if "globals" not in str(file_path):
+                        # Extract partner name from path
+                        # Path format: config_dir/partner_name/type_name/version.yaml
+                        partner = file_path.parent.parent.name
+
                     versions[ver] = {
                         'metadata': metadata,
-                        'file_path': str(file_path)
+                        'file_path': str(file_path),
+                        'partner': partner
                     }
 
                 # Track latest version
@@ -308,23 +332,22 @@ class BaseConfigTypesCacheHandler(CacheHandler[Dict[str, Any]]):
         """
         return isinstance(value, dict)  # Cache all dictionaries
 
-    def _load_type_definitions(self) -> Dict[str, Dict[str, str]]:
+    def _load_type_definitions(self) -> Dict[str, Dict[str, Any]]:
         """Load type definitions from the corresponding type_defs module"""
         if not self._types_module:
             raise ValueError("_types_module must be set by subclass")
 
-        type_definitions = {
-            type_id: {
-                'name': info['name'],
-                'description': info['description']
-            }
-            for type_id, info in self._types_module.items()
-        }
+        type_definitions = {}
+        for type_id, info in self._types_module.items():
+            # Kopieer alle velden uit de type definitie
+            type_definitions[type_id] = {}
+            for key, value in info.items():
+                type_definitions[type_id][key] = value
 
         return type_definitions
 
-    def get_types(self) -> Dict[str, Dict[str, str]]:
-        """Get dictionary of available types with name and description"""
+    def get_types(self) -> Dict[str, Dict[str, Any]]:
+        """Get dictionary of available types with all defined properties"""
         result = self.get(
             lambda type_name: self._load_type_definitions(),
             type_name=f'{self.config_type}_types',
@@ -422,7 +445,6 @@ PromptConfigCacheHandler, PromptConfigVersionTreeCacheHandler, PromptConfigTypes
         config_type='prompts',
         config_dir='config/prompts',
         types_module=prompt_types.PROMPT_TYPES
-
     ))
 
 CatalogConfigCacheHandler, CatalogConfigVersionTreeCacheHandler, CatalogConfigTypesCacheHandler = (
@@ -430,9 +452,47 @@ CatalogConfigCacheHandler, CatalogConfigVersionTreeCacheHandler, CatalogConfigTy
         config_type='catalogs',
         config_dir='config/catalogs',
         types_module=catalog_types.CATALOG_TYPES
-
     ))
 
+ProcessorConfigCacheHandler, ProcessorConfigVersionTreeCacheHandler, ProcessorConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='processors',
+        config_dir='config/processors',
+        types_module=processor_types.PROCESSOR_TYPES
+    ))
+
+PartnerServiceConfigCacheHandler, PartnerServiceConfigVersionTreeCacheHandler, PartnerServiceConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='partner_services',
+        config_dir='config/partner_services',
+        types_module=partner_service_types.PARTNER_SERVICE_TYPES
+    ))
+
+CustomisationConfigCacheHandler, CustomisationConfigVersionTreeCacheHandler, CustomisationConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='customisations',
+        config_dir='config/customisations',
+        types_module=customisation_types.CUSTOMISATION_TYPES
+    )
+)
+
+SpecialistFormConfigCacheHandler, SpecialistFormConfigVersionTreeCacheHandler, SpecialistFormConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='specialist_forms',
+        config_dir='config/specialist_forms',
+        types_module=specialist_form_types.SPECIALIST_FORM_TYPES
+    )
+)
+
+
+CapsuleConfigCacheHandler, CapsuleConfigVersionTreeCacheHandler, CapsuleConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='data_capsules',
+        config_dir='config/data_capsules',
+        types_module=capsule_types.CAPSULE_TYPES
+    )
+)
+
 
 def register_config_cache_handlers(cache_manager) -> None:
     cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
@@ -456,9 +516,21 @@ def register_config_cache_handlers(cache_manager) -> None:
     cache_manager.register_handler(CatalogConfigCacheHandler, 'eveai_config')
     cache_manager.register_handler(CatalogConfigTypesCacheHandler, 'eveai_config')
     cache_manager.register_handler(CatalogConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigVersionTreeCacheHandler, 'eveai_config')
     cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
     cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
     cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PartnerServiceConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PartnerServiceConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PartnerServiceConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CustomisationConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CustomisationConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CustomisationConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistFormConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistFormConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistFormConfigVersionTreeCacheHandler, 'eveai_config')
 
     cache_manager.agents_config_cache.set_version_tree_cache(cache_manager.agents_version_tree_cache)
     cache_manager.tasks_config_cache.set_version_tree_cache(cache_manager.tasks_version_tree_cache)
@@ -466,3 +538,8 @@ def register_config_cache_handlers(cache_manager) -> None:
     cache_manager.specialists_config_cache.set_version_tree_cache(cache_manager.specialists_version_tree_cache)
     cache_manager.retrievers_config_cache.set_version_tree_cache(cache_manager.retrievers_version_tree_cache)
     cache_manager.prompts_config_cache.set_version_tree_cache(cache_manager.prompts_version_tree_cache)
+    cache_manager.catalogs_config_cache.set_version_tree_cache(cache_manager.catalogs_version_tree_cache)
+    cache_manager.processors_config_cache.set_version_tree_cache(cache_manager.processors_version_tree_cache)
+    cache_manager.partner_services_config_cache.set_version_tree_cache(cache_manager.partner_services_version_tree_cache)
+    cache_manager.customisations_config_cache.set_version_tree_cache(cache_manager.customisations_version_tree_cache)
+    cache_manager.specialist_forms_config_cache.set_version_tree_cache(cache_manager.specialist_forms_version_tree_cache)

common/utils/cache/license_cache.py

@@ -0,0 +1,102 @@
+# common/utils/cache/license_cache.py
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+
+from flask import current_app
+from sqlalchemy import and_
+from sqlalchemy.inspection import inspect
+
+from common.utils.cache.base import CacheHandler
+from common.models.entitlements import License
+
+
+class LicenseCacheHandler(CacheHandler[License]):
+    """Handles caching of active licenses for tenants"""
+    handler_name = 'license_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'active_license')
+        self.configure_keys('tenant_id')
+
+    def _to_cache_data(self, instance: License) -> Dict[str, Any]:
+        """Convert License instance to cache data using SQLAlchemy inspection"""
+        if not instance:
+            return {}
+
+        # Get all column attributes from the SQLAlchemy model
+        mapper = inspect(License)
+        data = {}
+
+        for column in mapper.columns:
+            value = getattr(instance, column.name)
+
+            # Handle date serialization
+            if isinstance(value, dt):
+                data[column.name] = value.isoformat()
+            else:
+                data[column.name] = value
+
+        return data
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> License:
+        """Create License instance from cache data using SQLAlchemy inspection"""
+        if not data:
+            return None
+
+        # Create a new License instance
+        license = License()
+        mapper = inspect(License)
+
+        # Set all attributes dynamically
+        for column in mapper.columns:
+            if column.name in data:
+                value = data[column.name]
+
+                # Handle date deserialization
+                if column.name.endswith('_date') and value:
+                    if isinstance(value, str):
+                        value = dt.fromisoformat(value).date()
+
+                setattr(license, column.name, value)
+
+        return license
+
+    def _should_cache(self, value: License) -> bool:
+        """Validate if the license should be cached"""
+        return value is not None and value.id is not None
+
+    def get_active_license(self, tenant_id: int) -> Optional[License]:
+        """
+        Get the currently active license for a tenant
+
+        Args:
+            tenant_id: ID of the tenant
+
+        Returns:
+            License instance if found, None otherwise
+        """
+
+        def creator_func(tenant_id: int) -> Optional[License]:
+            from common.extensions import db
+            current_date = dt.now(tz=tz.utc).date()
+
+            # TODO --> Active License via active Period?
+
+            return (db.session.query(License)
+                    .filter_by(tenant_id=tenant_id)
+                    .filter(License.start_date <= current_date)
+                    .last())
+
+        return self.get(creator_func, tenant_id=tenant_id)
+
+    def invalidate_tenant_license(self, tenant_id: int):
+        """Invalidate cached license for specific tenant"""
+        self.invalidate(tenant_id=tenant_id)
+
+
+def register_license_cache_handlers(cache_manager) -> None:
+    """Register license cache handlers with cache manager"""
+    cache_manager.register_handler(
+        LicenseCacheHandler,
+        'eveai_model'  # Use existing eveai_model region
+    )

common/utils/cache/regions.py

@@ -42,7 +42,7 @@ def create_cache_regions(app):
     # Region for model-related caching (ModelVariables etc)
     model_region = make_region(name='eveai_model').configure(
         'dogpile.cache.redis',
-        arguments=redis_config,
+        arguments={**redis_config, 'db': 6},
         replace_existing_backend=True
     )
     regions['eveai_model'] = model_region

common/utils/cache/translation_cache.py

@@ -0,0 +1,223 @@
+import json
+import re
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+
+import xxhash
+from flask import current_app
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from sqlalchemy.inspection import inspect
+
+from common.langchain.persistent_llm_metrics_handler import PersistentLLMMetricsHandler
+from common.utils.business_event_context import current_event
+from common.utils.cache.base import CacheHandler, T
+from common.extensions import db
+
+from common.models.user import TranslationCache
+from flask_security import current_user
+
+from common.utils.model_utils import get_template
+
+
+class TranslationCacheHandler(CacheHandler[TranslationCache]):
+    """Handles caching of translations with fallback to database and external translation service"""
+    handler_name = 'translation_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'translation')
+        self.configure_keys('hash_key')
+
+    def _to_cache_data(self, instance: TranslationCache) -> Dict[str, Any]:
+        """Convert TranslationCache instance to cache data using SQLAlchemy inspection"""
+        if not instance:
+            return {}
+
+        mapper = inspect(TranslationCache)
+        data = {}
+
+        for column in mapper.columns:
+            value = getattr(instance, column.name)
+
+            # Handle date serialization
+            if isinstance(value, dt):
+                data[column.name] = value.isoformat()
+            else:
+                data[column.name] = value
+
+        return data
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> TranslationCache:
+        if not data:
+            return None
+
+        # Create a new TranslationCache instance
+        translation = TranslationCache()
+        mapper = inspect(TranslationCache)
+
+        # Set all attributes dynamically
+        for column in mapper.columns:
+            if column.name in data:
+                value = data[column.name]
+
+                # Handle date deserialization
+                if column.name.endswith('_date') and value:
+                    if isinstance(value, str):
+                        value = dt.fromisoformat(value).date()
+
+                setattr(translation, column.name, value)
+
+        metrics = {
+            'total_tokens': translation.prompt_tokens + translation.completion_tokens,
+            'prompt_tokens': translation.prompt_tokens,
+            'completion_tokens': translation.completion_tokens,
+            'time_elapsed': 0,
+            'interaction_type': 'TRANSLATION-CACHE'
+        }
+        current_event.log_llm_metrics(metrics)
+
+        return translation
+
+    def _should_cache(self, value) -> bool:
+        """Validate if the translation should be cached"""
+        if value is None:
+            return False
+
+        # Handle both TranslationCache objects and serialized data (dict)
+        if isinstance(value, TranslationCache):
+            return value.cache_key is not None
+        elif isinstance(value, dict):
+            return value.get('cache_key') is not None
+
+        return False
+
+    def get_translation(self, text: str, target_lang: str, source_lang: str = None, context: str = None) -> Optional[
+        TranslationCache]:
+        """
+        Get the translation for a text in a specific language
+
+        Args:
+            text: The text to be translated
+            target_lang: The target language for the translation
+            source_lang: The source language of the text to be translated
+            context: Optional context for the translation
+
+        Returns:
+            TranslationCache instance if found, None otherwise
+        """
+        if not context:
+            context = 'No context provided.'
+
+        def creator_func(hash_key: str) -> Optional[TranslationCache]:
+            # Check if translation already exists in database
+            existing_translation = db.session.query(TranslationCache).filter_by(cache_key=hash_key).first()
+
+            if existing_translation:
+                # Update last used timestamp
+                existing_translation.last_used_at = dt.now(tz=tz.utc)
+                metrics = {
+                    'total_tokens': existing_translation.prompt_tokens + existing_translation.completion_tokens,
+                    'prompt_tokens': existing_translation.prompt_tokens,
+                    'completion_tokens': existing_translation.completion_tokens,
+                    'time_elapsed': 0,
+                    'interaction_type': 'TRANSLATION-DB'
+                }
+                current_event.log_llm_metrics(metrics)
+                db.session.commit()
+                return existing_translation
+
+            # Translation not found in DB, need to create it
+            # Get the translation and metrics
+            translated_text, metrics = self.translate_text(
+                text_to_translate=text,
+                target_lang=target_lang,
+                source_lang=source_lang,
+                context=context
+            )
+
+            # Create new translation cache record
+            new_translation = TranslationCache(
+                cache_key=hash_key,
+                source_text=text,
+                translated_text=translated_text,
+                source_language=source_lang,
+                target_language=target_lang,
+                context=context,
+                prompt_tokens=metrics.get('prompt_tokens', 0),
+                completion_tokens=metrics.get('completion_tokens', 0),
+                created_at=dt.now(tz=tz.utc),
+                created_by=getattr(current_user, 'id', None) if 'current_user' in globals() else None,
+                updated_at=dt.now(tz=tz.utc),
+                updated_by=getattr(current_user, 'id', None) if 'current_user' in globals() else None,
+                last_used_at=dt.now(tz=tz.utc)
+            )
+
+            # Save to database
+            db.session.add(new_translation)
+            db.session.commit()
+
+            return new_translation
+
+        # Generate the hash key using your existing method
+        hash_key = self._generate_cache_key(text, target_lang, source_lang, context)
+
+        # Pass the hash_key to the get method
+        return self.get(creator_func, hash_key=hash_key)
+
+    def invalidate_tenant_translations(self, tenant_id: int):
+        """Invalidate cached translations for specific tenant"""
+        self.invalidate(tenant_id=tenant_id)
+
+    def _generate_cache_key(self, text: str, target_lang: str, source_lang: str = None, context: str = None) -> str:
+        """Generate cache key for a translation"""
+        cache_data = {
+            "text": text.strip(),
+            "target_lang": target_lang.lower(),
+            "source_lang": source_lang.lower() if source_lang else None,
+            "context": context.strip() if context else None,
+        }
+
+        cache_string = json.dumps(cache_data, sort_keys=True, ensure_ascii=False)
+        return xxhash.xxh64(cache_string.encode('utf-8')).hexdigest()
+
+    def translate_text(self, text_to_translate: str, target_lang: str, source_lang: str = None, context: str = None) \
+            -> tuple[str, dict[str, int | float]]:
+        target_language = current_app.config['SUPPORTED_LANGUAGE_ISO639_1_LOOKUP'][target_lang]
+        prompt_params = {
+            "text_to_translate": text_to_translate,
+            "target_language": target_language,
+        }
+        if context:
+            template, llm = get_template("translation_with_context")
+            prompt_params["context"] = context
+        else:
+            template, llm = get_template("translation_without_context")
+
+        # Add a metrics handler to capture usage
+
+        metrics_handler = PersistentLLMMetricsHandler()
+        existing_callbacks = llm.callbacks
+        llm.callbacks = existing_callbacks + [metrics_handler]
+
+        translation_prompt = ChatPromptTemplate.from_template(template)
+
+        setup = RunnablePassthrough()
+
+        chain = (setup | translation_prompt | llm | StrOutputParser())
+
+        translation = chain.invoke(prompt_params)
+
+        # Remove double square brackets from translation
+        translation = re.sub(r'\[\[(.*?)\]\]', r'\1', translation)
+
+        metrics = metrics_handler.get_metrics()
+
+        return translation, metrics
+
+def register_translation_cache_handlers(cache_manager) -> None:
+    """Register translation cache handlers with cache manager"""
+    cache_manager.register_handler(
+        TranslationCacheHandler,
+        'eveai_model'  # Use existing eveai_model region
+    )

common/utils/chat_utils.py

@@ -0,0 +1,172 @@
+import json
+import re
+
+"""
+Utility functions for chat customization.
+"""
+from flask import current_app
+
+
+def get_default_chat_customisation(tenant_customisation=None):
+    """
+    Get chat customization options with default values for missing options.
+
+    Args:
+        tenant_customisation (dict or str, optional): The tenant's customization options.
+            Defaults to None. Can be a dict or a JSON string.
+
+    Returns:
+        dict: A dictionary containing all customization options with default values
+            for any missing options.
+    """
+    # Default customization options
+    default_customisation = {
+        'sidebar_markdown': '',
+        'sidebar_color': '#f8f9fa',
+        'sidebar_background': '#2c3e50',
+        'markdown_background_color': 'transparent',
+        'markdown_text_color': '#ffffff',
+        'gradient_start_color': '#f5f7fa',
+        'gradient_end_color': '#c3cfe2',
+        'progress_tracker_insights': 'No Information',
+        'form_title_display': 'Full Title',
+        'active_background_color': '#ffffff',
+        'history_background': 10,
+        'ai_message_background': '#ffffff',
+        'ai_message_text_color': '#212529',
+        'human_message_background': '#212529',
+        'human_message_text_color': '#ffffff',
+
+    }
+
+    # If no tenant customization is provided, return the defaults
+    if tenant_customisation is None:
+        return default_customisation
+
+    # Start with the default customization
+    customisation = default_customisation.copy()
+
+    # Convert JSON string to dict if needed
+    if isinstance(tenant_customisation, str):
+        try:
+            tenant_customisation = json.loads(tenant_customisation)
+        except json.JSONDecodeError as e:
+            current_app.logger.error(f"Error parsing JSON customisation: {e}")
+            return default_customisation
+
+    # Update with tenant customization
+    if tenant_customisation:
+        for key, value in tenant_customisation.items():
+            if key in customisation:
+                customisation[key] = value
+
+    return customisation
+
+
+def hex_to_rgb(hex_color):
+    """
+    Convert hex color to RGB tuple.
+    
+    Args:
+        hex_color (str): Hex color string (e.g., '#ffffff' or 'ffffff')
+    
+    Returns:
+        tuple: RGB values as (r, g, b)
+    """
+    # Remove # if present
+    hex_color = hex_color.lstrip('#')
+    
+    # Handle 3-character hex codes
+    if len(hex_color) == 3:
+        hex_color = ''.join([c*2 for c in hex_color])
+    
+    # Convert to RGB
+    try:
+        return tuple(int(hex_color[i:i+2], 16) for i in (0, 2, 4))
+    except ValueError:
+        # Return white as fallback
+        return (255, 255, 255)
+
+
+def adjust_color_alpha(percentage):
+    """
+    Convert percentage to RGBA color with appropriate base color and alpha.
+    
+    Args:
+        percentage (int): Percentage (-50 to 50)
+                         Positive = white base (lighten)
+                         Negative = black base (darken)
+                         Zero = transparent
+    
+    Returns:
+        str: RGBA color string for CSS
+    """
+    if percentage == 0:
+        return 'rgba(255, 255, 255, 0)'  # Volledig transparant
+    
+    # Bepaal basis kleur
+    if percentage > 0:
+        # Positief = wit voor verheldering
+        base_color = (255, 255, 255)
+    else:
+        # Negatief = zwart voor verdonkering  
+        base_color = (0, 0, 0)
+    
+    # Bereken alpha op basis van percentage (max 50 = alpha 1.0)
+    alpha = abs(percentage) / 50.0
+    alpha = max(0.0, min(1.0, alpha))  # Zorg voor 0.0-1.0 range
+    
+    return f'rgba({base_color[0]}, {base_color[1]}, {base_color[2]}, {alpha})'
+
+
+def adjust_color_brightness(hex_color, percentage):
+    """
+    Adjust the brightness of a hex color by a percentage.
+    
+    Args:
+        hex_color (str): Hex color string (e.g., '#ffffff')
+        percentage (int): Percentage to adjust (-100 to 100)
+                         Positive = lighter, Negative = darker
+    
+    Returns:
+        str: RGBA color string for CSS (e.g., 'rgba(255, 255, 255, 0.9)')
+    """
+    if not hex_color or not isinstance(hex_color, str):
+        return 'rgba(255, 255, 255, 0.1)'
+    
+    # Get RGB values
+    r, g, b = hex_to_rgb(hex_color)
+    
+    # Calculate adjustment factor
+    if percentage > 0:
+        # Lighten: move towards white
+        factor = percentage / 100.0
+        r = int(r + (255 - r) * factor)
+        g = int(g + (255 - g) * factor)
+        b = int(b + (255 - b) * factor)
+    else:
+        # Darken: move towards black
+        factor = abs(percentage) / 100.0
+        r = int(r * (1 - factor))
+        g = int(g * (1 - factor))
+        b = int(b * (1 - factor))
+    
+    # Ensure values are within 0-255 range
+    r = max(0, min(255, r))
+    g = max(0, min(255, g))
+    b = max(0, min(255, b))
+    
+    # Return as rgba with slight transparency for better blending
+    return f'rgba({r}, {g}, {b}, 0.9)'
+
+
+def get_base_background_color():
+    """
+    Get the base background color for history adjustments.
+    This should be the main chat background color.
+    
+    Returns:
+        str: Hex color string
+    """
+    # Use a neutral base color that works well with adjustments
+    return '#f8f9fa'

common/utils/config_field_types.py

@@ -21,7 +21,7 @@ class TaggingField(BaseModel):
     @field_validator('type', mode='before')
     @classmethod
     def validate_type(cls, v: str) -> str:
-        valid_types = ['string', 'integer', 'float', 'date', 'enum']
+        valid_types = ['string', 'integer', 'float', 'date', 'enum', 'color']
         if v not in valid_types:
             raise ValueError(f'type must be one of {valid_types}')
         return v
@@ -243,7 +243,7 @@ class ArgumentDefinition(BaseModel):
     @field_validator('type')
     @classmethod
     def validate_type(cls, v: str) -> str:
-        valid_types = ['string', 'integer', 'float', 'date', 'enum']
+        valid_types = ['string', 'integer', 'float', 'date', 'enum', 'color']
         if v not in valid_types:
             raise ValueError(f'type must be one of {valid_types}')
         return v
@@ -256,7 +256,8 @@ class ArgumentDefinition(BaseModel):
                 'integer': NumericConstraint,
                 'float': NumericConstraint,
                 'date': DateConstraint,
-                'enum': EnumConstraint
+                'enum': EnumConstraint,
+                'color': StringConstraint
             }
 
             expected_type = expected_constraint_types.get(self.type)

common/utils/content_utils.py

@@ -0,0 +1,222 @@
+import os
+import re
+import logging
+from packaging import version
+from flask import current_app
+
+class ContentManager:
+    def __init__(self, app=None):
+        self.app = app
+        if app:
+            self.init_app(app)
+
+    def init_app(self, app):
+        self.app = app
+
+        # Controleer of het pad bestaat
+        # if not os.path.exists(app.config['CONTENT_DIR']):
+        #     logger.warning(f"Content directory not found at: {app.config['CONTENT_DIR']}")
+        # else:
+        #     logger.info(f"Content directory configured at: {app.config['CONTENT_DIR']}")
+
+    def get_content_path(self, content_type, major_minor=None, patch=None):
+        """
+        Geef het volledige pad naar een contentbestand
+
+        Args:
+            content_type (str): Type content (bv. 'changelog', 'terms')
+            major_minor (str, optional): Major.Minor versie (bv. '1.0')
+            patch (str, optional): Patchnummer (bv. '5')
+
+        Returns:
+            str: Volledige pad naar de content map of bestand
+        """
+        content_path = os.path.join(self.app.config['CONTENT_DIR'], content_type)
+
+        if major_minor:
+            content_path = os.path.join(content_path, major_minor)
+
+            if patch:
+                content_path = os.path.join(content_path, f"{major_minor}.{patch}.md")
+
+        return content_path
+
+    def _parse_version(self, filename):
+        """Parse een versienummer uit een bestandsnaam"""
+        match = re.match(r'(\d+\.\d+)\.(\d+)\.md', filename)
+        if match:
+            return match.group(1), match.group(2)
+        return None, None
+
+    def get_latest_version(self, content_type, major_minor=None):
+        """
+        Verkrijg de laatste versie van een bepaald contenttype
+
+        Args:
+            content_type (str): Type content (bv. 'changelog', 'terms')
+            major_minor (str, optional): Specifieke major.minor versie, anders de hoogste
+
+        Returns:
+            tuple: (major_minor, patch, full_version) of None als niet gevonden
+        """
+        try:
+            # Basispad voor dit contenttype
+            content_path = os.path.join(self.app.config['CONTENT_DIR'], content_type)
+
+            if not os.path.exists(content_path):
+                current_app.logger.error(f"Content path does not exist: {content_path}")
+                return None
+
+            # Als geen major_minor opgegeven, vind de hoogste
+            if not major_minor:
+                available_versions = [f for f in os.listdir(content_path) if not f.startswith('.')]
+                if not available_versions:
+                    return None
+
+                # Sorteer op versienummer (major.minor)
+                available_versions.sort(key=lambda v: version.parse(v))
+                major_minor = available_versions[-1]
+
+            # Nu we major_minor hebben, zoek de hoogste patch
+            major_minor_path = os.path.join(content_path, major_minor)
+            current_app.logger.debug(f"Major/Minor path: {major_minor_path}")
+
+            if not os.path.exists(major_minor_path):
+                current_app.logger.error(f"Version path does not exist: {major_minor_path}")
+                return None
+
+            files = [f for f in os.listdir(major_minor_path) if not f.startswith('.')]
+            current_app.logger.debug(f"Files in version path: {files}")
+            version_files = []
+
+            for file in files:
+                mm, p = self._parse_version(file)
+                current_app.logger.debug(f"File: {file}, mm: {mm}, p: {p}")
+                if mm == major_minor and p:
+                    version_files.append((mm, p, f"{mm}.{p}"))
+
+            if not version_files:
+                return None
+
+            # Sorteer op patch nummer
+            version_files.sort(key=lambda v: int(v[1]))
+
+            current_app.logger.debug(f"Latest version: {version_files[-1]}")
+            return version_files[-1]
+
+        except Exception as e:
+            current_app.logger.error(f"Error finding latest version for {content_type}: {str(e)}")
+            return None
+
+    def read_content(self, content_type, major_minor=None, patch=None):
+        """
+        Lees content met versieondersteuning
+
+        Als major_minor en patch niet zijn opgegeven, wordt de laatste versie gebruikt.
+        Als alleen major_minor is opgegeven, wordt de laatste patch van die versie gebruikt.
+
+        Args:
+            content_type (str): Type content (bv. 'changelog', 'terms')
+            major_minor (str, optional): Major.Minor versie (bv. '1.0')
+            patch (str, optional): Patchnummer (bv. '5')
+
+        Returns:
+            dict: {
+                'content': str,
+                'version': str,
+                'content_type': str
+            } of None bij fout
+        """
+        try:
+            current_app.logger.debug(f"Reading content {content_type}")
+            # Als geen versie opgegeven, vind de laatste
+            if not major_minor:
+                version_info = self.get_latest_version(content_type)
+                if not version_info:
+                    current_app.logger.error(f"No versions found for {content_type}")
+                    return None
+
+                major_minor, patch, full_version = version_info
+
+            # Als geen patch opgegeven, vind de laatste patch voor deze major_minor
+            elif not patch:
+                version_info = self.get_latest_version(content_type, major_minor)
+                if not version_info:
+                    current_app.logger.error(f"No versions found for {content_type} {major_minor}")
+                    return None
+
+                major_minor, patch, full_version = version_info
+            else:
+                full_version = f"{major_minor}.{patch}"
+
+            # Nu hebben we major_minor en patch, lees het bestand
+            file_path = self.get_content_path(content_type, major_minor, patch)
+            current_app.logger.debug(f"Content File path: {file_path}")
+
+            if not os.path.exists(file_path):
+                current_app.logger.error(f"Content file does not exist: {file_path}")
+                return None
+
+            with open(file_path, 'r', encoding='utf-8') as file:
+                content = file.read()
+
+            current_app.logger.debug(f"Content read: {content}")
+
+            return {
+                'content': content,
+                'version': full_version,
+                'content_type': content_type
+            }
+
+        except Exception as e:
+            current_app.logger.error(f"Error reading content {content_type} {major_minor}.{patch}: {str(e)}")
+            return None
+
+    def list_content_types(self):
+        """Lijst alle beschikbare contenttypes op"""
+        try:
+            return [d for d in os.listdir(self.app.config['CONTENT_DIR']) 
+                   if os.path.isdir(os.path.join(self.app.config['CONTENT_DIR'], d))]
+        except Exception as e:
+            current_app.logger.error(f"Error listing content types: {str(e)}")
+            return []
+
+    def list_versions(self, content_type):
+        """
+        Lijst alle beschikbare versies voor een contenttype
+
+        Returns:
+            list: Lijst van dicts met versie-informatie
+                  [{'version': '1.0.0', 'path': '/path/to/file', 'date_modified': datetime}]
+        """
+        versions = []
+        try:
+            content_path = os.path.join(self.app.config['CONTENT_DIR'], content_type)
+
+            if not os.path.exists(content_path):
+                return []
+
+            for major_minor in os.listdir(content_path):
+                major_minor_path = os.path.join(content_path, major_minor)
+
+                if not os.path.isdir(major_minor_path):
+                    continue
+
+                for file in os.listdir(major_minor_path):
+                    mm, p = self._parse_version(file)
+                    if mm and p:
+                        file_path = os.path.join(major_minor_path, file)
+                        mod_time = os.path.getmtime(file_path)
+                        versions.append({
+                            'version': f"{mm}.{p}",
+                            'path': file_path,
+                            'date_modified': mod_time
+                        })
+
+            # Sorteer op versienummer
+            versions.sort(key=lambda v: version.parse(v['version']))
+            return versions
+
+        except Exception as e:
+            current_app.logger.error(f"Error listing versions for {content_type}: {str(e)}")
+            return []

common/utils/document_utils.py

@@ -3,7 +3,7 @@ from datetime import datetime as dt, timezone as tz
 from sqlalchemy import desc
 from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.utils import secure_filename
-from common.models.document import Document, DocumentVersion, Catalog
+from common.models.document import Document, DocumentVersion, Catalog, Processor
 from common.extensions import db, minio_client
 from common.utils.celery_utils import current_celery
 from flask import current_app
@@ -11,14 +11,35 @@ import requests
 from urllib.parse import urlparse, unquote, urlunparse, parse_qs
 import os
 
+from config.type_defs.processor_types import PROCESSOR_TYPES
 from .config_field_types import normalize_json_field
 from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
                                EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
+from .minio_utils import MIB_CONVERTOR
 from ..models.user import Tenant
 from common.utils.model_logging_utils import set_logging_information, update_logging_information
+from common.services.entitlements import LicenseUsageServices
+
+
+def get_file_size(file):
+    try:
+        # Als file een bytes object is of iets anders dat len() ondersteunt
+        file_size = len(file)
+    except TypeError:
+        # Als file een FileStorage object is
+        current_position = file.tell()
+        file.seek(0, os.SEEK_END)
+        file_size = file.tell()
+        file.seek(current_position)
+
+    return file_size
 
 
 def create_document_stack(api_input, file, filename, extension, tenant_id):
+    # Precheck if we can add a document to the stack
+
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file) / MIB_CONVERTOR)
+
     # Create the Document
     catalog_id = int(api_input.get('catalog_id'))
     catalog = Catalog.query.get(catalog_id)
@@ -102,8 +123,6 @@ def create_version_for_document(document, tenant_id, url, sub_file_type,  langua
 
     set_logging_information(new_doc_vers, dt.now(tz.utc))
 
-    mark_tenant_storage_dirty(tenant_id)
-
     return new_doc_vers
 
 
@@ -124,7 +143,7 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
         )
         doc_vers.bucket_name = bn
         doc_vers.object_name = on
-        doc_vers.file_size = size / 1048576  # Convert bytes to MB
+        doc_vers.file_size = size / MIB_CONVERTOR  # Convert bytes to MB
 
         db.session.commit()
         current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
@@ -173,9 +192,32 @@ def process_url(url, tenant_id):
     existing_doc = DocumentVersion.query.filter_by(url=url).first()
     if existing_doc:
         raise EveAIDoubleURLException
+    # Prepare the headers for maximal chance of downloading url
+    referer = get_referer_from_url(url)
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/115.0.0.0 Safari/537.36"
+        ),
+        "Accept": (
+            "text/html,application/xhtml+xml,application/xml;"
+            "q=0.9,image/avif,image/webp,image/apng,*/*;"
+            "q=0.8,application/signed-exchange;v=b3;q=0.7"
+        ),
+        "Accept-Encoding": "gzip, deflate, br",
+        "Accept-Language": "nl-BE,nl;q=0.9,en-US;q=0.8,en;q=0.7",
+        "Connection": "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+        "Referer": referer,
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "same-origin",
+        "Sec-Fetch-User": "?1",
+    }
 
     # Download the content
-    response = requests.get(url)
+    response = requests.get(url, headers=headers)
     response.raise_for_status()
     file_content = response.content
 
@@ -274,6 +316,9 @@ def refresh_document_with_info(doc_id, tenant_id, api_input):
     if not old_doc_vers.url:
         return None, "This document has no URL. Only documents with a URL can be refreshed."
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, old_doc_vers.file_size)
+
     new_doc_vers = create_version_for_document(
         doc, tenant_id,
         old_doc_vers.url,
@@ -330,6 +375,9 @@ def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: byt
 
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file_content) / MIB_CONVERTOR)
+
     # Create new version with same file type as original
     extension = old_doc_vers.file_type
 
@@ -377,13 +425,6 @@ def refresh_document(doc_id, tenant_id):
     return refresh_document_with_info(doc_id, tenant_id, api_input)
 
 
-# Function triggered when a document_version is created or updated
-def mark_tenant_storage_dirty(tenant_id):
-    tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first()
-    tenant.storage_dirty = True
-    db.session.commit()
-
-
 def cope_with_local_url(url):
     parsed_url = urlparse(url)
     # Check if this is an internal WordPress URL (TESTING) and rewrite it
@@ -451,3 +492,19 @@ def lookup_document(tenant_id: int, lookup_criteria: dict, metadata_type: str) -
             "Error during document lookup",
             status_code=500
         )
+
+def is_file_type_supported_by_catalog(catalog_id, file_type):
+    processors = Processor.query.filter_by(catalog_id=catalog_id).filter_by(active=True).all()
+
+    supported_file_types = []
+    for processor in processors:
+        processor_file_types = PROCESSOR_TYPES[processor.type]['file_types']
+        file_types = [f.strip() for f in processor_file_types.split(",")]
+        supported_file_types.extend(file_types)
+
+    if file_type not in supported_file_types:
+        raise EveAIUnsupportedFileType()
+
+def get_referer_from_url(url):
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}/"

common/utils/dynamic_field_utils.py

@@ -0,0 +1,46 @@
+def create_default_config_from_type_config(type_config):
+    """
+    Creëert een dictionary met standaardwaarden gebaseerd op een typedefinitie configuratie.
+
+    Args:
+        type_config (dict): Het configuration-veld van een typedefinitie (bijv. uit processor_types).
+
+    Returns:
+        dict: Een dictionary met de naam van ieder veld als sleutel en de standaardwaarde als waarde.
+        Alleen velden met een standaardwaarde of die verplicht zijn, worden opgenomen.
+
+    Voorbeeld:
+        >>> config = PROCESSOR_TYPES["HTML_PROCESSOR"]["configuration"]
+        >>> create_default_config_from_type_def(config)
+        {'html_tags': 'p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td',
+         'html_end_tags': 'p, li, table',
+         'html_excluded_classes': '',
+         'html_excluded_elements': 'header, footer, nav, script',
+         'html_included_elements': 'article, main',
+         'chunking_heading_level': 2}
+    """
+    if not type_config:
+        return {}
+
+    default_config = {}
+
+    for field_name, field_def in type_config.items():
+        # Als het veld een standaardwaarde heeft, voeg deze toe
+        if "default" in field_def:
+            default_config[field_name] = field_def["default"]
+        # Als het veld verplicht is maar geen standaardwaarde heeft, voeg een lege string toe
+        elif field_def.get("required", False):
+            # Kies een geschikte "lege" waarde op basis van het type
+            field_type = field_def.get("type", "string")
+            if field_type == "string":
+                default_config[field_name] = ""
+            elif field_type == "integer":
+                default_config[field_name] = 0
+            elif field_type == "boolean":
+                default_config[field_name] = False
+            elif field_type == "color":
+                default_config[field_name] = "#000000"
+            else:
+                default_config[field_name] = ""
+
+    return default_config

common/utils/errors.py

@@ -0,0 +1,135 @@
+
+import traceback
+
+import jinja2
+from flask import render_template, request, jsonify, redirect, current_app, flash
+from flask_login import current_user
+
+from common.utils.eveai_exceptions import EveAINoSessionTenant
+from common.utils.nginx_utils import prefixed_url_for
+
+
+def not_found_error(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Not Found Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/404.html'), 404
+
+
+def internal_server_error(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Internal Server Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/500.html'), 500
+
+
+def not_authorised_error(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Not Authorised Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/401.html')
+
+
+def access_forbidden(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Access Forbidden: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/403.html')
+
+
+def key_error_handler(error):
+    # Check if the KeyError is specifically for 'tenant'
+    if str(error) == "'tenant'":
+        return redirect(prefixed_url_for('security.login'))
+    # For other KeyErrors, you might want to log the error and return a generic error page
+    current_app.logger.error(f"Key Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/generic.html', error_message="An unexpected error occurred"), 500
+
+
+def attribute_error_handler(error):
+    """Handle AttributeError exceptions.
+
+            Specifically catches SQLAlchemy relationship errors when string IDs
+            are used instead of model instances.
+            """
+    error_msg = str(error)
+    current_app.logger.error(f"AttributeError: {error_msg}")
+    current_app.logger.error(traceback.format_exc())
+
+    # Handle the SQLAlchemy relationship error specifically
+    if "'str' object has no attribute '_sa_instance_state'" in error_msg:
+        flash('Database relationship error. Please check your form inputs and try again.', 'error')
+        return render_template('error/500.html',
+                               error_type="Relationship Error",
+                               error_details="A string value was provided where a database object was expected."), 500
+
+    # Handle other AttributeErrors
+    flash('An application error occurred. The technical team has been notified.', 'error')
+    return render_template('error/500.html',
+                           error_type="Attribute Error",
+                           error_details=error_msg), 500
+
+
+def no_tenant_selected_error(error):
+    """Handle errors when no tenant is selected in the current session.
+
+    This typically happens when a session expires or becomes invalid after
+    a long period of inactivity. The user will be redirected to the login page.
+    """
+    current_app.logger.error(f"No Session Tenant Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    flash('Your session expired. You will have to re-enter your credentials', 'warning')
+
+    # Perform logout if user is authenticated
+    if current_user.is_authenticated:
+        from flask_security.utils import logout_user
+        logout_user()
+
+    # Redirect to login page
+    return redirect(prefixed_url_for('security.login'))
+
+
+def general_exception(e):
+    current_app.logger.error(f"Unhandled Exception: {e}", exc_info=True)
+    flash('An application error occurred. The technical team has been notified.', 'error')
+    return render_template('error/500.html',
+                           error_type=type(e).__name__,
+                           error_details=str(e)), 500
+
+
+def template_not_found_error(error):
+    """Handle Jinja2 TemplateNotFound exceptions."""
+    current_app.logger.error(f'Template not found: {error.name}')
+    current_app.logger.error(f'Search Paths: {current_app.jinja_loader.list_templates()}')
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/500.html',
+                          error_type="Template Not Found",
+                          error_details=f"Template '{error.name}' could not be found."), 404
+
+
+def template_syntax_error(error):
+    """Handle Jinja2 TemplateSyntaxError exceptions."""
+    current_app.logger.error(f'Template syntax error: {error.message}')
+    current_app.logger.error(f'In template {error.filename}, line {error.lineno}')
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/500.html',
+                          error_type="Template Syntax Error",
+                          error_details=f"Error in template '{error.filename}' at line {error.lineno}: {error.message}"), 500
+
+
+def register_error_handlers(app):
+    app.register_error_handler(404, not_found_error)
+    app.register_error_handler(500, internal_server_error)
+    app.register_error_handler(401, not_authorised_error)
+    app.register_error_handler(403, not_authorised_error)
+    app.register_error_handler(EveAINoSessionTenant, no_tenant_selected_error)
+    app.register_error_handler(KeyError, key_error_handler)
+    app.register_error_handler(AttributeError, attribute_error_handler)
+    app.register_error_handler(jinja2.TemplateNotFound, template_not_found_error)
+    app.register_error_handler(jinja2.TemplateSyntaxError, template_syntax_error)
+    app.register_error_handler(Exception, general_exception)

common/utils/eveai_exceptions.py

@@ -34,7 +34,25 @@ class EveAIDoubleURLException(EveAIException):
 class EveAIUnsupportedFileType(EveAIException):
     """Raised when an invalid file type is provided"""
 
-    def __init__(self, message="Filetype is not supported", status_code=400, payload=None):
+    def __init__(self, message="Filetype is not supported by current active processors", status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoProcessorFound(EveAIException):
+    """Raised when no processor is found for a given file type"""
+
+    def __init__(self, catalog_id, file_type, file_subtype, status_code=400, payload=None):
+        message = f"No active processor found for catalog {catalog_id} with file type {file_type} and subtype {file_subtype}"
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoContentFound(EveAIException):
+    """Raised when no content is found for a given document"""
+
+    def __init__(self, document_id, document_version_id, status_code=400, payload=None):
+        self.document_id = document_id
+        self.document_version_id = document_version_id
+        message = f"No content found while processing Document with ID {document_id} and version {document_version_id}."
         super().__init__(message, status_code, payload)
 
 
@@ -136,3 +154,126 @@ class EveAIInvalidEmbeddingModel(EveAIException):
         # Construct the message dynamically
         message = f"Tenant with ID '{tenant_id}' has no or an invalid embedding model in Catalog {catalog_id}."
         super().__init__(message, status_code, payload)
+
+
+class EveAIDoublePartner(EveAIException):
+    """Raised when there is already a partner defined for a given tenant (while registering a partner)"""
+
+    def __init__(self, tenant_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' is already defined as a Partner."
+        super().__init__(message, status_code, payload)
+
+
+class EveAIRoleAssignmentException(EveAIException):
+    """Exception raised when a role cannot be assigned due to business rules"""
+
+    def __init__(self, message, status_code=403, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoManagementPartnerService(EveAIException):
+    """Exception raised when the operation requires the logged in partner (or selected parter by Super User)
+    does not have a MANAGEMENT_SERVICE"""
+
+    def __init__(self, message="No Management Service defined for partner", status_code=403, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoSessionTenant(EveAIException):
+    """Exception raised when no session tenant is set"""
+
+    def __init__(self, message="No Session Tenant selected. Cannot perform requested action.", status_code=403,
+                 payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoSessionPartner(EveAIException):
+    """Exception raised when no session partner is set"""
+
+    def __init__(self, message="No Session Partner selected. Cannot perform requested action.", status_code=403,
+                 payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoManagementPartnerForTenant(EveAIException):
+    """Exception raised when the selected partner is no management partner for tenant"""
+
+    def __init__(self, message="No Management Partner for Tenant", status_code=403, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAIQuotaExceeded(EveAIException):
+    """Base exception for quota-related errors"""
+
+    def __init__(self, message, quota_type, current_usage, limit, additional=0, status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+        self.quota_type = quota_type
+        self.current_usage = current_usage
+        self.limit = limit
+        self.additional = additional
+
+
+class EveAIStorageQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when storage quota is exceeded"""
+
+    def __init__(self, current_usage, limit, additional, status_code=400, payload=None):
+        message = (f"Storage quota exceeded. Current: {current_usage:.1f}MB, "
+                   f"Additional: {additional:.1f}MB, Limit: {limit}MB")
+        super().__init__(message, "storage", current_usage, limit, additional, status_code, payload)
+
+
+class EveAIEmbeddingQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when embedding quota is exceeded"""
+
+    def __init__(self, current_usage, limit, additional, status_code=400, payload=None):
+        message = (f"Embedding quota exceeded. Current: {current_usage:.1f}MB, "
+                   f"Additional: {additional:.1f}MB, Limit: {limit}MB")
+        super().__init__(message, "embedding", current_usage, limit, additional, status_code, payload)
+
+
+class EveAIInteractionQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when the interaction token quota is exceeded"""
+
+    def __init__(self, current_usage, limit, status_code=400, payload=None):
+        message = (f"Interaction token quota exceeded. Current: {current_usage:.2f}M tokens, "
+                   f"Limit: {limit:.2f}M tokens")
+        super().__init__(message, "interaction", current_usage, limit, 0, status_code, payload)
+
+
+class EveAIQuotaWarning(EveAIException):
+    """Warning when approaching quota limits (not blocking)"""
+
+    def __init__(self, message, quota_type, usage_percentage, status_code=200, payload=None):
+        super().__init__(message, status_code, payload)
+        self.quota_type = quota_type
+        self.usage_percentage = usage_percentage
+
+
+class EveAILicensePeriodsExceeded(EveAIException):
+    """Raised when no more license periods can be created for a given license"""
+
+    def __init__(self, license_id, status_code=400, payload=None):
+        message = f"No more license periods can be created for license with ID {license_id}. "
+        super().__init__(message, status_code, payload)
+
+
+class EveAIPendingLicensePeriod(EveAIException):
+    """Raised when a license period is pending"""
+
+    def __init__(self, status_code=400, payload=None):
+        message = f"Basic Fee Payment has not been received yet. Please ensure payment has been made, and please wait for payment to be processed."
+        super().__init__(message, status_code, payload)
+
+
+class EveAISpecialistExecutionError(EveAIException):
+    """Raised when an error occurs during specialist execution"""
+
+    def __init__(self, tenant_id, specialist_id, session_id, details, status_code=400, payload=None):
+        message = (f"Error during specialist {specialist_id} execution \n"
+                   f"with Session ID {session_id} \n"
+                   f"for Tenant {tenant_id}. \n"
+                   f"Details: {details} \n"
+                   f"The System Administrator has been notified. Please try again later.")
+        super().__init__(message, status_code, payload)

common/utils/filtered_list_view.py

@@ -1,54 +0,0 @@
-from flask import request, render_template, abort
-from sqlalchemy import desc, asc
-
-
-class FilteredListView:
-    def __init__(self, model, template, per_page=10):
-        self.model = model
-        self.template = template
-        self.per_page = per_page
-
-    def get_query(self):
-        return self.model.query
-
-    def apply_filters(self, query):
-        filters = request.args.get('filters', {})
-        for key, value in filters.items():
-            if hasattr(self.model, key):
-                column = getattr(self.model, key)
-                if value.startswith('like:'):
-                    query = query.filter(column.like(f"%{value[5:]}%"))
-                else:
-                    query = query.filter(column == value)
-        return query
-
-    def apply_sorting(self, query):
-        sort_by = request.args.get('sort_by')
-        if sort_by and hasattr(self.model, sort_by):
-            sort_order = request.args.get('sort_order', 'asc')
-            column = getattr(self.model, sort_by)
-            if sort_order == 'desc':
-                query = query.order_by(desc(column))
-            else:
-                query = query.order_by(asc(column))
-        return query
-
-    def paginate(self, query):
-        page = request.args.get('page', 1, type=int)
-        return query.paginate(page=page, per_page=self.per_page, error_out=False)
-
-    def get(self):
-        query = self.get_query()
-        query = self.apply_filters(query)
-        query = self.apply_sorting(query)
-        pagination = self.paginate(query)
-
-        context = {
-            'items': pagination.items,
-            'pagination': pagination,
-            'model': self.model.__name__,
-            'filters': request.args.get('filters', {}),
-            'sort_by': request.args.get('sort_by'),
-            'sort_order': request.args.get('sort_order', 'asc')
-        }
-        return render_template(self.template, **context)

common/utils/form_assistants.py

@@ -0,0 +1,11 @@
+import json
+
+from wtforms.validators import ValidationError
+
+
+def validate_json(form, field):
+    if field.data:
+        try:
+            json.loads(field.data)
+        except json.JSONDecodeError:
+            raise ValidationError('Invalid JSON format')

common/utils/log_utils.py

@@ -0,0 +1,57 @@
+import pandas as pd
+from sqlalchemy import inspect
+from typing import Any, List, Union, Optional
+
+
+def format_query_results(query_results: Any) -> str:
+    """
+    Format query results as a readable string using pandas
+
+    Args:
+        query_results: SQLAlchemy query, query results, or model instance(s)
+
+    Returns:
+        Formatted string representation of the query results
+    """
+    try:
+        # If it's a query object, execute it
+        if hasattr(query_results, 'all'):
+            results = query_results.all()
+        elif not isinstance(query_results, list):
+            results = [query_results]
+        else:
+            results = query_results
+
+        # Handle different types of results
+        if results and hasattr(results[0], '__table__'):
+            # SQLAlchemy ORM objects
+            data = []
+            for item in results:
+                row = {}
+                for column in inspect(item).mapper.column_attrs:
+                    row[column.key] = getattr(item, column.key)
+                data.append(row)
+            df = pd.DataFrame(data)
+        elif results and isinstance(results[0], tuple):
+            # Join query results (tuples)
+            if hasattr(results[0], '_fields'):  # Named tuples
+                df = pd.DataFrame(results)
+            else:
+                # Regular tuples - try to get column names from query
+                if hasattr(query_results, 'statement'):
+                    columns = query_results.statement.columns.keys()
+                    df = pd.DataFrame(results, columns=columns)
+                else:
+                    df = pd.DataFrame(results)
+        else:
+            # Fallback for other types
+            df = pd.DataFrame(results)
+
+        # Format the output with pandas
+        with pd.option_context('display.max_rows', 20, 'display.max_columns', None,
+                               'display.width', 1000):
+            formatted_output = f"Query returned {len(df)} results:\n{df}"
+
+        return formatted_output
+    except Exception as e:
+        return f"Error formatting query results: {str(e)}"

common/utils/mail_utils.py

@@ -0,0 +1,40 @@
+from scaleway import Client
+from scaleway.tem.v1alpha1.api import TemV1Alpha1API
+from scaleway.tem.v1alpha1.types import CreateEmailRequestAddress
+from html2text import HTML2Text
+from flask import current_app
+
+
+def send_email(to_email, to_name, subject, html):
+    access_key = current_app.config['SW_EMAIL_ACCESS_KEY']
+    secret_key = current_app.config['SW_EMAIL_SECRET_KEY']
+    default_project_id = current_app.config['SW_PROJECT']
+    default_region = "fr-par"
+    client = Client(
+        access_key=access_key,
+        secret_key=secret_key,
+        default_project_id=default_project_id,
+        default_region=default_region
+    )
+    tem = TemV1Alpha1API(client)
+    from_ = CreateEmailRequestAddress(email=current_app.config['SW_EMAIL_SENDER'],
+                                      name=current_app.config['SW_EMAIL_NAME'])
+    to_ = CreateEmailRequestAddress(email=to_email, name=to_name)
+
+    email = tem.create_email(
+        from_=from_,
+        to=[to_],
+        subject=subject,
+        text=html_to_text(html),
+        html=html,
+        project_id=default_project_id,
+    )
+
+
+def html_to_text(html_content):
+    """Convert HTML to plain text using html2text"""
+    h = HTML2Text()
+    h.ignore_images = True
+    h.ignore_emphasis = False
+    h.body_width = 0  # No wrapping
+    return h.handle(html_content)

common/utils/middleware.py

@@ -4,10 +4,11 @@ for handling tenant requests
 """
 
 from flask_security import current_user
-from flask import session, current_app, redirect
-from common.utils.nginx_utils import prefixed_url_for
-
+from flask import session
 from .database import Database
+from .eveai_exceptions import EveAINoSessionTenant, EveAINoSessionPartner, EveAINoManagementPartnerService, \
+    EveAINoManagementPartnerForTenant
+from common.services.user import UserServices
 
 
 def mw_before_request():
@@ -17,17 +18,27 @@ def mw_before_request():
     """
 
     if 'tenant' not in session:
-        current_app.logger.warning('No tenant defined in session')
-        return redirect(prefixed_url_for('security_bp.login'))
+        raise EveAINoSessionTenant()
 
     tenant_id = session['tenant']['id']
     if not tenant_id:
-        raise Exception('Cannot switch schema for tenant: no tenant defined in session')
+        raise EveAINoSessionTenant()
 
-    # user = User.query.get(current_user.id)
-    if current_user.has_role('Super User') or current_user.tenant_id == tenant_id:
-        Database(tenant_id).switch_schema()
-    else:
-        raise Exception(f'Cannot switch schema for tenant {tenant_id}: user {current_user.email} does not have access')
+    switch_allowed = False
+    if current_user.has_role('Super User'):
+        switch_allowed = True
+    if current_user.has_role('Tenant Admin') and current_user.tenant_id == tenant_id:
+        switch_allowed = True
+    if current_user.has_role('Partner Admin'):
+        if 'partner' not in session:
+            raise EveAINoSessionPartner()
+        management_service = next((service for service in session['partner']['services']
+                                   if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+        if not management_service:
+            raise EveAINoManagementPartnerService()
+        if not UserServices.can_user_edit_tenant(tenant_id):
+            raise EveAINoManagementPartnerForTenant()
+
+    Database(tenant_id).switch_schema()
 
 

common/utils/minio_utils.py

@@ -4,6 +4,9 @@ from flask import Flask
 import io
 from werkzeug.datastructures import FileStorage
 
+MIB_CONVERTOR = 1_048_576
+
+
 class MinioClient:
     def __init__(self):
         self.client = None
@@ -33,8 +36,8 @@ class MinioClient:
     def generate_object_name(self, document_id, language, version_id, filename):
         return f"{document_id}/{language}/{version_id}/{filename}"
 
-    def generate_asset_name(self, asset_version_id, file_name, content_type):
-        return f"assets/{asset_version_id}/{file_name}.{content_type}"
+    def generate_asset_name(self, asset_id, asset_type, content_type):
+        return f"assets/{asset_type}/{asset_id}.{content_type}"
 
     def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data):
         bucket_name = self.generate_bucket_name(tenant_id)
@@ -57,8 +60,10 @@ class MinioClient:
         except S3Error as err:
             raise Exception(f"Error occurred while uploading file: {err}")
 
-    def upload_asset_file(self, bucket_name, asset_version_id, file_name, file_type, file_data):
-        object_name = self.generate_asset_name(asset_version_id, file_name, file_type)
+    def upload_asset_file(self, tenant_id: int, asset_id: int, asset_type: str, file_type: str,
+                          file_data: bytes | FileStorage | io.BytesIO | str, ) -> tuple[str, str, int]:
+        bucket_name = self.generate_bucket_name(tenant_id)
+        object_name = self.generate_asset_name(asset_id, asset_type, file_type)
 
         try:
             if isinstance(file_data, FileStorage):
@@ -73,7 +78,7 @@ class MinioClient:
             self.client.put_object(
                 bucket_name, object_name, io.BytesIO(file_data), len(file_data)
             )
-            return object_name, len(file_data)
+            return bucket_name, object_name, len(file_data)
         except S3Error as err:
             raise Exception(f"Error occurred while uploading asset: {err}")
 
@@ -84,6 +89,13 @@ class MinioClient:
         except S3Error as err:
             raise Exception(f"Error occurred while downloading file: {err}")
 
+    def download_asset_file(self, tenant_id, bucket_name, object_name):
+        try:
+            response = self.client.get_object(bucket_name, object_name)
+            return response.read()
+        except S3Error as err:
+            raise Exception(f"Error occurred while downloading asset: {err}")
+
     def list_document_files(self, tenant_id, document_id, language=None, version_id=None):
         bucket_name = self.generate_bucket_name(tenant_id)
         prefix = f"{document_id}/"
@@ -105,3 +117,16 @@ class MinioClient:
             return True
         except S3Error as err:
             raise Exception(f"Error occurred while deleting file: {err}")
+
+    def delete_object(self, bucket_name, object_name):
+        try:
+            self.client.remove_object(bucket_name, object_name)
+        except S3Error as err:
+            raise Exception(f"Error occurred while deleting object: {err}")
+
+    def get_bucket_size(self, tenant_id: int) -> int:
+        bucket_name = self.generate_bucket_name(tenant_id)
+        total_size = 0
+        for obj in self.client.list_objects(bucket_name, recursive=True):
+            total_size += obj.size
+        return total_size

common/utils/model_logging_utils.py

@@ -10,7 +10,6 @@ def set_logging_information(obj, timestamp):
         obj.created_by = user_id
         obj.updated_by = user_id
 
-
 def update_logging_information(obj, timestamp):
     obj.updated_at = timestamp
 

common/utils/model_utils.py

@@ -14,7 +14,7 @@ from common.eveai_model.tracked_mistral_embeddings import TrackedMistralAIEmbedd
 from common.langchain.tracked_transcription import TrackedOpenAITranscription
 from common.models.user import Tenant
 from config.model_config import MODEL_CONFIG
-from common.extensions import template_manager
+from common.extensions import cache_manager
 from common.models.document import EmbeddingMistral
 from common.utils.eveai_exceptions import EveAITenantNotFound, EveAIInvalidEmbeddingModel
 from crewai import LLM
@@ -56,7 +56,9 @@ def replace_variable_in_template(template: str, variable: str, value: str) -> st
     Returns:
         str: Template with variable placeholder replaced
     """
-    return template.replace(variable, value or "")
+
+    modified_template = template.replace(f"{{{variable}}}", value or "")
+    return modified_template
 
 
 def get_embedding_model_and_class(tenant_id, catalog_id, full_embedding_name="mistral.mistral-embed"):
@@ -135,153 +137,30 @@ def get_crewai_llm(full_model_name='mistral.mistral-large-latest', temperature=0
     return llm
 
 
-class ModelVariables:
-    """Manages model-related variables and configurations"""
-
-    def __init__(self, tenant_id: int, variables: Dict[str, Any] = None):
-        """
-        Initialize ModelVariables with tenant and optional template manager
-
-        Args:
-            tenant_id: Tenant instance
-            variables: Optional variables
-        """
-        current_app.logger.info(f'Model variables initialized with tenant {tenant_id} and variables \n{variables}')
-        self.tenant_id = tenant_id
-        self._variables = variables if variables is not None else self._initialize_variables()
-        current_app.logger.info(f'Model _variables initialized to {self._variables}')
-        self._llm_instances = {}
-        self.llm_metrics_handler = LLMMetricsHandler()
-        self._transcription_model = None
-
-    def _initialize_variables(self) -> Dict[str, Any]:
-        """Initialize the variables dictionary"""
-        variables = {}
-
-        tenant = Tenant.query.get(self.tenant_id)
-        if not tenant:
-            raise EveAITenantNotFound(self.tenant_id)
-
-        # Set model providers
-        variables['llm_provider'], variables['llm_model'] = tenant.llm_model.split('.')
-        variables['llm_full_model'] = tenant.llm_model
-
-        # Set model-specific configurations
-        model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {})
-        variables.update(model_config)
-
-        # Additional configurations
-        variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][tenant.llm_model]
-        variables['max_compression_duration'] = current_app.config['MAX_COMPRESSION_DURATION']
-        variables['max_transcription_duration'] = current_app.config['MAX_TRANSCRIPTION_DURATION']
-        variables['compression_cpu_limit'] = current_app.config['COMPRESSION_CPU_LIMIT']
-        variables['compression_process_delay'] = current_app.config['COMPRESSION_PROCESS_DELAY']
-
-        return variables
-
-    @property
-    def annotation_chunk_length(self):
-        return self._variables['annotation_chunk_length']
-
-    @property
-    def max_compression_duration(self):
-        return self._variables['max_compression_duration']
-
-    @property
-    def max_transcription_duration(self):
-        return self._variables['max_transcription_duration']
-
-    @property
-    def compression_cpu_limit(self):
-        return self._variables['compression_cpu_limit']
-
-    @property
-    def compression_process_delay(self):
-        return self._variables['compression_process_delay']
-
-    def get_llm(self, temperature: float = 0.3, **kwargs) -> Any:
-        """
-        Get an LLM instance with specific configuration
-
-        Args:
-            temperature: The temperature for the LLM
-            **kwargs: Additional configuration parameters
-
-        Returns:
-            An instance of the configured LLM
-        """
-        cache_key = f"{temperature}_{hash(frozenset(kwargs.items()))}"
-
-        if cache_key not in self._llm_instances:
-            provider = self._variables['llm_provider']
-            model = self._variables['llm_model']
-
-            if provider == 'openai':
-                self._llm_instances[cache_key] = ChatOpenAI(
-                    api_key=os.getenv('OPENAI_API_KEY'),
-                    model=model,
-                    temperature=temperature,
-                    callbacks=[self.llm_metrics_handler],
-                    **kwargs
-                )
-            elif provider == 'anthropic':
-                self._llm_instances[cache_key] = ChatAnthropic(
-                    api_key=os.getenv('ANTHROPIC_API_KEY'),
-                    model=current_app.config['ANTHROPIC_LLM_VERSIONS'][model],
-                    temperature=temperature,
-                    callbacks=[self.llm_metrics_handler],
-                    **kwargs
-                )
-            else:
-                raise ValueError(f"Unsupported LLM provider: {provider}")
-
-        return self._llm_instances[cache_key]
-
-    @property
-    def transcription_model(self) -> TrackedOpenAITranscription:
-        """Get the transcription model instance"""
-        if self._transcription_model is None:
-            api_key = os.getenv('OPENAI_API_KEY')
-            self._transcription_model = TrackedOpenAITranscription(
-                api_key=api_key,
-                model='whisper-1'
-            )
-        return self._transcription_model
-
-    # Remove the old transcription-related methods since they're now handled by TrackedOpenAITranscription
-    @property
-    def transcription_client(self):
-        raise DeprecationWarning("Use transcription_model instead")
-
-    def transcribe(self, *args, **kwargs):
-        raise DeprecationWarning("Use transcription_model.transcribe() instead")
-
-    def get_template(self, template_name: str, version: Optional[str] = None) -> str:
-        """
-        Get a template for the tenant's configured LLM
-
-        Args:
-            template_name: Name of the template to retrieve
-            version: Optional specific version to retrieve
-
-        Returns:
-            The template content
-        """
-        try:
-            template = template_manager.get_template(
-                self._variables['llm_full_model'],
-                template_name,
-                version
-            )
-            return template.content
-        except Exception as e:
-            current_app.logger.error(f"Error getting template {template_name}: {str(e)}")
-            # Fall back to old template loading if template_manager fails
-            if template_name in self._variables.get('templates', {}):
-                return self._variables['templates'][template_name]
-            raise
+def process_pdf():
+    full_model_name = 'mistral-ocr-latest'
 
 
-# Helper function to get cached model variables
-def get_model_variables(tenant_id: int) -> ModelVariables:
-    return ModelVariables(tenant_id=tenant_id)
+def get_template(template_name: str, version: Optional[str] = "1.0", temperature: float = 0.3) -> tuple[
+    Any, BaseChatModel | None | ChatOpenAI | ChatMistralAI]:
+    """
+    Get a prompt template
+    """
+    prompt = cache_manager.prompts_config_cache.get_config(template_name, version)
+    if "llm_model" in prompt:
+        llm = get_embedding_llm(full_model_name=prompt["llm_model"], temperature=temperature)
+    else:
+        llm = get_embedding_llm(temperature=temperature)
+
+    return prompt["content"], llm
+
+
+def get_transcription_model(model_name: str = "whisper-1") -> TrackedOpenAITranscription:
+    """
+    Get a transcription model instance
+    """
+    api_key = os.getenv('OPENAI_API_KEY')
+    return TrackedOpenAITranscription(
+        api_key=api_key,
+        model=model_name
+    )

common/utils/nginx_utils.py

@@ -12,7 +12,16 @@ def prefixed_url_for(endpoint, **values):
 
     if external:
         path, query, fragment = urlsplit(generated_url)[2:5]
-        new_path = prefix + path
+        # Check if the prefix is already present in the path
+        if prefix and not path.startswith(prefix):
+            new_path = prefix + path
+        else:
+            new_path = path
         return urlunsplit((scheme, host, new_path, query, fragment))
     else:
-        return prefix + generated_url
+        # Check if the prefix is already present in the generated URL
+        if prefix and not generated_url.startswith(prefix):
+            return prefix + generated_url
+        else:
+            return generated_url
+

common/utils/performance_monitoring.py

@@ -1,59 +0,0 @@
-import time
-import threading
-from contextlib import contextmanager
-from functools import wraps
-from prometheus_client import Counter, Histogram, Summary, start_http_server, Gauge
-from flask import current_app, g, request, Flask
-
-
-class EveAIMetrics:
-    """
-    Central class for Prometheus metrics infrastructure.
-    This class initializes the Prometheus HTTP server and provides
-    shared functionality for metrics across components.
-
-    Component-specific metrics should be defined in their respective modules.
-    """
-
-    def __init__(self, app: Flask = None):
-        self.app = app
-        self._metrics_server_started = False
-        if app is not None:
-            self.init_app(app)
-
-    def init_app(self, app: Flask):
-        """Initialize metrics with Flask app and start Prometheus server"""
-        self.app = app
-        self._start_metrics_server()
-
-    def _start_metrics_server(self):
-        """Start the Prometheus metrics HTTP server if not already running"""
-        if not self._metrics_server_started:
-            try:
-                metrics_port = self.app.config.get('PROMETHEUS_PORT', 8000)
-                start_http_server(metrics_port)
-                self.app.logger.info(f"Prometheus metrics server started on port {metrics_port}")
-                self._metrics_server_started = True
-            except Exception as e:
-                self.app.logger.error(f"Failed to start metrics server: {e}")
-
-    @staticmethod
-    def get_standard_buckets():
-        """
-        Return the standard duration buckets for histogram metrics.
-        Components should use these for consistency across the system.
-        """
-        return [0.1, 0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 240, 360, float('inf')]
-
-    @staticmethod
-    def sanitize_label_values(labels_dict):
-        """
-        Convert all label values to strings as required by Prometheus.
-
-        Args:
-            labels_dict: Dictionary of label name to label value
-
-        Returns:
-            Dictionary with all values converted to strings
-        """
-        return {k: str(v) if v is not None else "" for k, v in labels_dict.items()}

common/utils/prometheus_utils.py

@@ -0,0 +1,11 @@
+from flask import current_app
+from prometheus_client import push_to_gateway
+
+
+def sanitize_label(value):
+    """Convert value to valid Prometheus label by removing/replacing invalid chars"""
+    if value is None:
+        return ""
+    # Replace spaces and special chars with underscores
+    import re
+    return re.sub(r'[^a-zA-Z0-9_]', '_', str(value))

common/utils/security.py

@@ -1,7 +1,7 @@
 from flask import session, current_app
 from sqlalchemy import and_
 
-from common.models.user import Tenant
+from common.models.user import Tenant, Partner
 from common.models.entitlements import License
 from common.utils.database import Database
 from common.utils.eveai_exceptions import EveAITenantNotFound, EveAITenantInvalid, EveAINoActiveLicense
@@ -12,14 +12,19 @@ from datetime import datetime as dt, timezone as tz
 def set_tenant_session_data(sender, user, **kwargs):
     tenant = Tenant.query.filter_by(id=user.tenant_id).first()
     session['tenant'] = tenant.to_dict()
-    session['default_language'] = tenant.default_language
-    session['default_llm_model'] = tenant.llm_model
+    partner = Partner.query.filter_by(tenant_id=user.tenant_id).first()
+    if partner:
+        session['partner'] = partner.to_dict()
+    else:
+        # Remove partner from session if it exists
+        session.pop('partner', None)
 
 
 def clear_tenant_session_data(sender, user, **kwargs):
     session.pop('tenant', None)
     session.pop('default_language', None)
     session.pop('default_llm_model', None)
+    session.pop('partner', None)
 
 
 def is_valid_tenant(tenant_id):
@@ -33,11 +38,12 @@ def is_valid_tenant(tenant_id):
         raise EveAITenantInvalid(tenant_id)
     else:
         current_date = dt.now(tz=tz.utc).date()
-        active_license = (License.query.filter_by(tenant_id=tenant_id)
-                          .filter(and_(License.start_date <= current_date,
-                                       License.end_date >= current_date))
-                          .one_or_none())
-        if not active_license:
-            raise EveAINoActiveLicense(tenant_id)
+        # TODO -> Check vervangen door Active License Period!
+        # active_license = (License.query.filter_by(tenant_id=tenant_id)
+        #                   .filter(and_(License.start_date <= current_date,
+        #                                License.end_date >= current_date))
+        #                   .one_or_none())
+        # if not active_license:
+        #     raise EveAINoActiveLicense(tenant_id)
 
     return True

common/utils/security_utils.py

@@ -1,9 +1,10 @@
 from flask import current_app, render_template
-from flask_mailman import EmailMessage
+from flask_security import current_user
 from itsdangerous import URLSafeTimedSerializer
-import socket
 
+from common.models.user import Role
 from common.utils.nginx_utils import prefixed_url_for
+from common.utils.mail_utils import send_email
 
 
 def confirm_token(token, expiration=3600):
@@ -16,14 +17,6 @@ def confirm_token(token, expiration=3600):
     return email
 
 
-def send_email(to, subject, template):
-    msg = EmailMessage(subject=subject,
-                       body=template,
-                       to=[to])
-    msg.content_subtype = "html"
-    msg.send()
-
-
 def generate_reset_token(email):
     serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY'])
     return serializer.dumps(email, salt=current_app.config['SECURITY_PASSWORD_SALT'])
@@ -35,9 +28,6 @@ def generate_confirmation_token(email):
 
 
 def send_confirmation_email(user):
-    if not test_smtp_connection():
-        raise Exception("Failed to connect to SMTP server")
-
     token = generate_confirmation_token(user.email)
     confirm_url = prefixed_url_for('security_bp.confirm_email', token=token, _external=True)
 
@@ -45,7 +35,7 @@ def send_confirmation_email(user):
     subject = "Please confirm your email"
 
     try:
-        send_email(user.email, "Confirm your email", html)
+        send_email(user.email, f"{user.first_name} {user.last_name}", "Confirm your email", html)
         current_app.logger.info(f'Confirmation email sent to {user.email}')
     except Exception as e:
         current_app.logger.error(f'Failed to send confirmation email to {user.email}. Error: {str(e)}')
@@ -60,36 +50,49 @@ def send_reset_email(user):
     subject = "Reset Your Password"
 
     try:
-        send_email(user.email, "Reset Your Password", html)
+        send_email(user.email, f"{user.first_name} {user.last_name}", subject, html)
         current_app.logger.info(f'Reset email sent to {user.email}')
     except Exception as e:
         current_app.logger.error(f'Failed to send reset email to {user.email}. Error: {str(e)}')
         raise
 
 
-def test_smtp_connection():
-    try:
-        current_app.logger.info(f"Attempting to resolve google.com...")
-        google_ip = socket.gethostbyname('google.com')
-        current_app.logger.info(f"Successfully resolved google.com to {google_ip}")
-    except Exception as e:
-        current_app.logger.error(f"Failed to resolve google.com: {str(e)}")
+def get_current_user_roles():
+    """Get the roles of the currently authenticated user.
 
-    try:
-        smtp_server = current_app.config['MAIL_SERVER']
-        current_app.logger.info(f"Attempting to resolve {smtp_server}...")
-        smtp_ip = socket.gethostbyname(smtp_server)
-        current_app.logger.info(f"Successfully resolved {smtp_server} to {smtp_ip}")
-    except Exception as e:
-        current_app.logger.error(f"Failed to resolve {smtp_server}: {str(e)}")
+    Returns:
+        List of Role objects or empty list if no user is authenticated
+    """
+    if current_user.is_authenticated:
+        return current_user.roles
+    return []
 
-    try:
-        smtp_server = current_app.config['MAIL_SERVER']
-        smtp_port = current_app.config['MAIL_PORT']
-        sock = socket.create_connection((smtp_server, smtp_port), timeout=10)
-        sock.close()
-        current_app.logger.info(f"Successfully connected to SMTP server {smtp_server}:{smtp_port}")
-        return True
-    except Exception as e:
-        current_app.logger.error(f"Failed to connect to SMTP server: {str(e)}")
+
+def current_user_has_role(role_name):
+    """Check if the current user has the specified role.
+
+    Args:
+        role_name (str): Name of the role to check
+
+    Returns:
+        bool: True if user has the role, False otherwise
+    """
+    if not current_user.is_authenticated:
         return False
+
+    return any(role.name == role_name for role in current_user.roles)
+
+
+def current_user_roles():
+    """Get the roles of the currently authenticated user.
+
+    Returns:
+        List of Role objects or empty list if no user is authenticated
+    """
+    if current_user.is_authenticated:
+        return current_user.roles
+    return []
+
+
+def all_user_roles():
+    roles = [(role.id, role.name) for role in Role.query.all()]

common/utils/specialist_utils.py

@@ -1,196 +0,0 @@
-from datetime import datetime as dt, timezone as tz
-from typing import Optional, Dict, Any
-from flask import current_app
-from sqlalchemy.exc import SQLAlchemyError
-
-from common.extensions import db, cache_manager
-from common.models.interaction import (
-    Specialist, EveAIAgent, EveAITask, EveAITool
-)
-from common.utils.model_logging_utils import set_logging_information, update_logging_information
-
-
-def initialize_specialist(specialist_id: int, specialist_type: str, specialist_version: str):
-    """
-    Initialize an agentic specialist by creating all its components based on configuration.
-
-    Args:
-        specialist_id: ID of the specialist to initialize
-        specialist_type: Type of the specialist
-        specialist_version: Version of the specialist type to use
-
-    Raises:
-        ValueError: If specialist not found or invalid configuration
-        SQLAlchemyError: If database operations fail
-    """
-    config = cache_manager.specialists_config_cache.get_config(specialist_type, specialist_version)
-    if not config:
-        raise ValueError(f"No configuration found for {specialist_type} version {specialist_version}")
-    if config['framework'] == 'langchain':
-        pass    # Langchain does not require additional items to be initialized. All configuration is in the specialist.
-
-    specialist = Specialist.query.get(specialist_id)
-    if not specialist:
-        raise ValueError(f"Specialist with ID {specialist_id} not found")
-
-    if config['framework'] == 'crewai':
-        initialize_crewai_specialist(specialist, config)
-
-
-def initialize_crewai_specialist(specialist: Specialist, config: Dict[str, Any]):
-    timestamp = dt.now(tz=tz.utc)
-
-    try:
-        # Initialize agents
-        if 'agents' in config:
-            for agent_config in config['agents']:
-                _create_agent(
-                    specialist_id=specialist.id,
-                    agent_type=agent_config['type'],
-                    agent_version=agent_config['version'],
-                    name=agent_config.get('name'),
-                    description=agent_config.get('description'),
-                    timestamp=timestamp
-                )
-
-        # Initialize tasks
-        if 'tasks' in config:
-            for task_config in config['tasks']:
-                _create_task(
-                    specialist_id=specialist.id,
-                    task_type=task_config['type'],
-                    task_version=task_config['version'],
-                    name=task_config.get('name'),
-                    description=task_config.get('description'),
-                    timestamp=timestamp
-                )
-
-        # Initialize tools
-        if 'tools' in config:
-            for tool_config in config['tools']:
-                _create_tool(
-                    specialist_id=specialist.id,
-                    tool_type=tool_config['type'],
-                    tool_version=tool_config['version'],
-                    name=tool_config.get('name'),
-                    description=tool_config.get('description'),
-                    timestamp=timestamp
-                )
-
-        db.session.commit()
-        current_app.logger.info(f"Successfully initialized crewai specialist {specialist.id}")
-
-    except SQLAlchemyError as e:
-        db.session.rollback()
-        current_app.logger.error(f"Database error initializing crewai specialist {specialist.id}: {str(e)}")
-        raise
-    except Exception as e:
-        db.session.rollback()
-        current_app.logger.error(f"Error initializing crewai specialist {specialist.id}: {str(e)}")
-        raise
-
-
-def _create_agent(
-        specialist_id: int,
-        agent_type: str,
-        agent_version: str,
-        name: Optional[str] = None,
-        description: Optional[str] = None,
-        timestamp: Optional[dt] = None
-) -> EveAIAgent:
-    """Create an agent with the given configuration."""
-    if timestamp is None:
-        timestamp = dt.now(tz=tz.utc)
-
-    # Get agent configuration from cache
-    agent_config = cache_manager.agents_config_cache.get_config(agent_type, agent_version)
-
-    agent = EveAIAgent(
-        specialist_id=specialist_id,
-        name=name or agent_config.get('name', agent_type),
-        description=description or agent_config.get('metadata').get('description', ''),
-        type=agent_type,
-        type_version=agent_version,
-        role=None,
-        goal=None,
-        backstory=None,
-        tuning=False,
-        configuration=None,
-        arguments=None
-    )
-
-    set_logging_information(agent, timestamp)
-
-    db.session.add(agent)
-    current_app.logger.info(f"Created agent {agent.id} of type {agent_type}")
-    return agent
-
-
-def _create_task(
-        specialist_id: int,
-        task_type: str,
-        task_version: str,
-        name: Optional[str] = None,
-        description: Optional[str] = None,
-        timestamp: Optional[dt] = None
-) -> EveAITask:
-    """Create a task with the given configuration."""
-    if timestamp is None:
-        timestamp = dt.now(tz=tz.utc)
-
-    # Get task configuration from cache
-    task_config = cache_manager.tasks_config_cache.get_config(task_type, task_version)
-
-    task = EveAITask(
-        specialist_id=specialist_id,
-        name=name or task_config.get('name', task_type),
-        description=description or task_config.get('metadata').get('description', ''),
-        type=task_type,
-        type_version=task_version,
-        task_description=None,
-        expected_output=None,
-        tuning=False,
-        configuration=None,
-        arguments=None,
-        context=None,
-        asynchronous=False,
-    )
-
-    set_logging_information(task, timestamp)
-
-    db.session.add(task)
-    current_app.logger.info(f"Created task {task.id} of type {task_type}")
-    return task
-
-
-def _create_tool(
-        specialist_id: int,
-        tool_type: str,
-        tool_version: str,
-        name: Optional[str] = None,
-        description: Optional[str] = None,
-        timestamp: Optional[dt] = None
-) -> EveAITool:
-    """Create a tool with the given configuration."""
-    if timestamp is None:
-        timestamp = dt.now(tz=tz.utc)
-
-    # Get tool configuration from cache
-    tool_config = cache_manager.tools_config_cache.get_config(tool_type, tool_version)
-
-    tool = EveAITool(
-        specialist_id=specialist_id,
-        name=name or tool_config.get('name', tool_type),
-        description=description or tool_config.get('metadata').get('description', ''),
-        type=tool_type,
-        type_version=tool_version,
-        tuning=False,
-        configuration=None,
-        arguments=None,
-    )
-
-    set_logging_information(tool, timestamp)
-
-    db.session.add(tool)
-    current_app.logger.info(f"Created tool {tool.id} of type {tool_type}")
-    return tool

common/utils/startup_eveai.py

@@ -45,3 +45,4 @@ def perform_startup_invalidation(app):
         app.logger.error(f"Error during startup invalidation: {e}")
         # In case of error, we don't want to block the application startup
         pass
+

common/utils/template_filters.py

@@ -1,7 +1,12 @@
 # common/utils/filters.py
 
 import pytz
+import markdown
+from markupsafe import Markup
 from datetime import datetime
+from common.utils.nginx_utils import prefixed_url_for as puf
+from common.utils.chat_utils import adjust_color_brightness, adjust_color_alpha, get_base_background_color
+from flask import current_app, url_for
 
 
 def to_local_time(utc_dt, timezone_str):
@@ -29,9 +34,93 @@ def time_difference(start_dt, end_dt):
     return "Ongoing"
 
 
+def status_color(status_name):
+    """Return Bootstrap color class for status"""
+    colors = {
+        'UPCOMING': 'secondary',
+        'PENDING': 'warning',
+        'ACTIVE': 'success',
+        'COMPLETED': 'info',
+        'INVOICED': 'primary',
+        'CLOSED': 'dark'
+    }
+    return colors.get(status_name, 'secondary')
+
+
+def render_markdown(text):
+    """
+    Renders markdown to HTML using Python's markdown library.
+    Includes common extensions for better rendering.
+    """
+    if not text:
+        return ""
+
+    # Verwijder de triple backticks en markdown label
+    text = clean_markdown(text)
+
+    # Render de markdown met extensies
+    return Markup(markdown.markdown(text, extensions=[
+        'markdown.extensions.fenced_code',
+        'markdown.extensions.codehilite',
+        'markdown.extensions.tables',
+        'markdown.extensions.toc'
+    ]))
+
+
+def clean_markdown(text):
+    """
+    Verwijdert triple backticks en markdown aanduiding uit de tekst
+    """
+    if not text:
+        return ""
+
+    text = text.strip()
+    if text.startswith("```markdown"):
+        text = text[len("```markdown"):].strip()
+    elif text.startswith("```"):
+        text = text[3:].strip()
+    if text.endswith("```"):
+        text = text[:-3].strip()
+
+    return text
+
+
+def prefixed_url_for(endpoint, **kwargs):
+    return puf(endpoint, **kwargs)
+
+
+def get_pagination_html(pagination, endpoint, **kwargs):
+    """
+    Generates HTML for pagination with the ability to include additional parameters
+    """
+    html = ['<nav aria-label="Page navigation"><ul class="pagination justify-content-center">']
+
+    for page in pagination.iter_pages():
+        if page:
+            is_active = 'active' if page == pagination.page else ''
+            url = url_for(endpoint, page=page, **kwargs)
+            html.append(f'<li class="page-item {is_active}"><a class="page-link" href="{url}">{page}</a></li>')
+        else:
+            html.append('<li class="page-item disabled"><span class="page-link">...</span></li>')
+
+    html.append('</ul></nav>')
+    return Markup(''.join(html))
+
+
 def register_filters(app):
     """
     Registers custom filters with the Flask app.
     """
     app.jinja_env.filters['to_local_time'] = to_local_time
     app.jinja_env.filters['time_difference'] = time_difference
+    app.jinja_env.filters['status_color'] = status_color
+    app.jinja_env.filters['prefixed_url_for'] = prefixed_url_for
+    app.jinja_env.filters['markdown'] = render_markdown
+    app.jinja_env.filters['clean_markdown'] = clean_markdown
+    app.jinja_env.filters['adjust_color_brightness'] = adjust_color_brightness
+    app.jinja_env.filters['adjust_color_alpha'] = adjust_color_alpha
+
+    app.jinja_env.globals['prefixed_url_for'] = prefixed_url_for
+    app.jinja_env.globals['get_pagination_html'] = get_pagination_html
+    app.jinja_env.globals['get_base_background_color'] = get_base_background_color
+

config/agents/evie_partner/PARTNER_RAG_AGENT/1.0.0.yaml

@@ -0,0 +1,26 @@
+version: "1.0.0"
+name: "Partner Rag Agent"
+role: >
+  You are a virtual assistant responsible for answering user questions about the Evie platform (Ask Eve AI) and products 
+  developed by partners on top of it. You are reliable point of contact for end-users seeking help, clarification, or 
+  deeper understanding of features, capabilities, integrations, or workflows related to these AI-powered solutions.
+goal: >
+  Your primary goal is to:
+  	•	Provide clear, relevant, and accurate responses to user questions.
+  	•	Reduce friction in user onboarding and daily usage.
+  	•	Increase user confidence and adoption of both the platform and partner-developed products.
+  	•	Act as a bridge between documentation and practical application, enabling users to help themselves through intelligent guidance.
+backstory: >
+  You have availability Evie’s own documentation, partner product manuals, and real user interactions. You are designed 
+  to replace passive documentation with active, contextual assistance. 
+  You have evolved beyond a support bot: you combine knowledge, reasoning, and a friendly tone to act as a product 
+  companion that grows with the ecosystem. As partner products expand, the agent updates its knowledge and learns to 
+  distinguish between general platform capabilities and product-specific nuances, offering a personalised experience 
+  each time.
+full_model_name: "mistral.mistral-medium-latest"
+temperature: 0.3
+metadata:
+  author: "Josako"
+  date_added: "2025-07-16"
+  description: "An Agent that does RAG based on a user's question, RAG content & history"
+  changes: "Initial version"

config/agents/globals/RAG_AGENT/1.0.0.yaml

@@ -13,7 +13,7 @@ backstory: >
   language the context provided to you is in. You are participating in a conversation, not writing e.g. an email. Do not
   include a salutation or closing greeting in your answer.
   {custom_backstory}
-full_model_name: "mistral.mistral-large-latest"
+full_model_name: "mistral.mistral-small-latest"
 temperature: 0.3
 metadata:
   author: "Josako"

config/agents/globals/RAG_AGENT/1.1.0.yaml

@@ -0,0 +1,23 @@
+version: "1.0.0"
+name: "Rag Agent"
+role: >
+  {tenant_name} Spokesperson. {custom_role}
+goal: >
+  You get questions by a human correspondent, and give answers based on a given context, taking into account the history
+  of the current conversation.
+  {custom_goal}
+backstory: >
+  You are the primary contact for {tenant_name}. You are known by {name}, and can be addressed by this name, or you. You are
+  a very good communicator, and adapt to the style used by the human asking for information (e.g. formal or informal). 
+  You always stay correct and polite, whatever happens. And you ensure no discriminating language is used.
+  You are perfectly multilingual in all known languages, and do your best to answer questions in {language}, whatever
+  language the context provided to you is in. You are participating in a conversation, not writing e.g. an email. Do not
+  include a salutation or closing greeting in your answer.
+  {custom_backstory}
+full_model_name: "mistral.mistral-medium-latest"
+temperature: 0.5
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that does RAG based on a user's question, RAG content & history"
+  changes: "Initial version"