Changelog for version 2.3.3-alfa

- Catalog Name Unique Constraint
  - Selection constraint to view processed document
  - remove tab from tenant overview

.gitignore

@@ -14,7 +14,6 @@ __pycache__
 **/__pycache__
 /.idea
 *.pyc
-*.pyc
 common/.DS_Store
 common/__pycache__/__init__.cpython-312.pyc
 common/__pycache__/extensions.cpython-312.pyc
@@ -42,3 +41,15 @@ migrations/public/.DS_Store
 scripts/.DS_Store
 scripts/__pycache__/run_eveai_app.cpython-312.pyc
 /eveai_repo.txt
+*repo.txt
+/docker/eveai_logs/
+/integrations/Wordpress/eveai_sync.zip
+/integrations/Wordpress/eveai-chat.zip
+/db_backups/
+/tests/interactive_client/specialist_client.log
+/.repopackignore
+/patched_packages/crewai/
+/docker/prometheus/data/
+/docker/grafana/data/
+/temp_requirements/
+/nginx/node_modules/

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/eveAI.iml

@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="Flask">
-    <option name="enabled" value="true" />
-  </component>
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/.venv" />
-      <excludeFolder url="file://$MODULE_DIR$/.venv2" />
-    </content>
-    <orderEntry type="jdk" jdkName="Python 3.12 (eveai_dev)" jdkType="Python SDK" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="TemplatesService">
-    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
-    <option name="TEMPLATE_FOLDERS">
-      <list>
-        <option value="$MODULE_DIR$/templates" />
-      </list>
-    </option>
-  </component>
-</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Black">
-    <option name="sdkName" value="Python 3.12 (eveai_tbd)" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (eveai_tbd)" project-jdk-type="Python SDK" />
-</project>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/TBD.iml" filepath="$PROJECT_DIR$/.idea/TBD.iml" />
-    </modules>
-  </component>
-</project>

.idea/sqldialects.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="SqlDialectMappings">
-    <file url="PROJECT" dialect="PostgreSQL" />
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

.python-version

@@ -1 +0,0 @@
-eveai_tbd

CHANGELOG.md

@@ -5,10 +5,195 @@ All notable changes to EveAI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [2.3.0-alfa]
 
 ### Added
-- For new features.
+- Introduction of Push Gateway for Prometheus
+- Introduction of Partner Models
+- Introduction of Tenant and Partner codes for more security
+- Introduction of 'Management Partner' type and additional 'Partner Admin'-role
+- Introduction of a technical services layer
+- Introduction of partner-specific configurations
+- Introduction of additional test environment
+- Introduction of strict no-overage usage
+- Introduction of LicensePeriod, Payments & Invoices
+- Introduction of Processed File Viewer
+- Introduction of Traicie Role Definition Specialist
+- Allow invocation of non-interactive specialists in administrative interface (eveai_app)
+- Introduction of advanced JSON editor
+- Introduction of ChatSession (Specialist Execution) follow-up in administrative interface
+- Introduce npm for javascript libraries usage and optimisations
+- Introduction of new top bar in administrative interface to show session defaults (removing old navbar buttons)
+- 
+
+### Changed
+- Add 'Register'-button to list views, replacing register menu-items
+- Add additional environment capabilities in docker
+- PDF Processor now uses Mistral OCR
+- Allow additional chunking mechanisms for very long chunks (in case of very large documents)
+- Allow for TrackedMistralAIEmbedding batching to allow for processing long documents
+- RAG & SPIN Specialist improvements
+- Move mail messaging from standard SMTP to Scaleway TEM mails
+- Improve mail layouts
+- Add functionality to add a default dictionary for dynamic forms
+- AI model choices defined by Ask Eve AI iso Tenant (replaces ModelVariables completely)
+- Improve HTML Processing
+- Pagination improvements
+- Update Material Kit Pro to latest version
+
+### Removed
+- Repopack implementation ==> Using PyCharm's new AI capabilities instead
+
+### Fixed
+- Synchronous vs Asynchronous behaviour in crewAI type specialists
+- Nasty dynamic boolean fields bug corrected
+- Several smaller bugfixes
+- Tasks & Tools editors finished
+
+### Security
+- In case of vulnerabilities.
+
+## [2.2.0-alfa]
+
+### Added
+- Mistral AI as main provider for embeddings, chains and specialists
+- Usage measuring for specialists
+- RAG from chain to specialist technology
+- Dossier catalog management possibilities added to eveai_app
+- Asset definition (Paused - other priorities)
+- Prometheus and Grafana
+- Add prometheus monitoring to business events
+- Asynchronous execution of specialists
+
+### Changed
+- Moved choice for AI providers / models to specialists and prompts
+- Improve RAG to not repeat historic answers
+- Fixed embedding model, no more choices allowed
+- clean url (of tracking parameters) before adding it to a catalog
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- Add Multiple URLs removed from menu
+- Old Specialist items removed from interaction menu
+- 
+
+### Fixed
+- Set default language when registering Documents or URLs.
+
+### Security
+- In case of vulnerabilities.
+
+## [2.1.0-alfa]
+
+### Added
+- Zapier Refresh Document
+- SPIN Specialist definition - from start to finish
+- Introduction of startup scripts in eveai_app
+- Caching for all configurations added
+- Caching for processed specialist configurations
+- Caching for specialist history
+- Augmented Specialist Editor, including Specialist graphic presentation
+- Introduction of specialist_execution_api, introducting SSE
+- Introduction of crewai framework for specialist implementation
+- Test app for testing specialists - also serves as a sample client application for SSE
+- 
+
+### Changed
+- Improvement of startup of applications using gevent, and better handling and scaling of multiple connections
+- STANDARD_RAG Specialist improvement
+- 
+
+### Deprecated
+- eveai_chat - using sockets - will be replaced with new specialist_execution_api and SSE
+
+## [2.0.1-alfa]
+
+### Added
+- Zapîer Integration (partial - only adding files).
+- Addition of general chunking parameters (chunking_heading_level and chunking_patterns)
+- Addition of DocX and markdown Processor Types
+
+### Changed
+- For changes in existing functionality.
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- Ensure the RAG Specialist is using the detailed_question
+- Wordpress Chat Plugin: languages dropdown filled again
+- OpenAI update - proxies no longer supported
+- Build & Release script for Wordpress Plugins (including end user download folder)
+
+### Security
+- In case of vulnerabilities.
+
+## [2.0.0-alfa]
+
+### Added
+- Introduction of dynamic Retrievers & Specialists
+- Introduction of dynamic Processors
+- Introduction of caching system
+- Introduction of a better template manager
+- Modernisation of external API/Socket authentication using projects
+- Creation of new eveai_chat WordPress plugin to support specialists
+
+### Changed
+- Update of eveai_sync WordPress plugin
+
+### Fixed
+- Set default language when registering Documents or URLs.
+
+### Security
+- Security improvements to Docker images
+
+## [1.0.14-alfa]
+
+### Added
+- New release script added to tag images with release number
+- Allow the addition of multiple types of Catalogs
+- Generic functionality to enable dynamic fields
+- Addition of Retrievers to allow for smart collection of information in Catalogs
+- Add dynamic fields to Catalog / Retriever / DocumentVersion
+
+### Changed
+- Processing parameters defined at Catalog level iso Tenant level
+- Reroute 'blank' paths to 'admin'
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- Set default language when registering Documents or URLs.
+
+### Security
+- In case of vulnerabilities.
+
+## [1.0.13-alfa]
+
+### Added
+- Finished Catalog introduction
+- Reinitialization of WordPress site for syncing
+
+### Changed
+- Modification of WordPress Sync Component
+- Cleanup of attributes in Tenant
+
+### Fixed
+- Overall bugfixes as result from the Catalog introduction
+
+## [1.0.12-alfa]
+
+### Added
+- Added Catalog functionality
 
 ### Changed
 - For changes in existing functionality.
@@ -25,6 +210,53 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Security
 - In case of vulnerabilities.
 
+## [1.0.11-alfa]
+
+### Added
+- License Usage Calculation realised
+- View License Usages
+- Celery Beat container added
+- First schedule in Celery Beat for calculating usage (hourly)
+
+### Changed
+- repopack can now split for different components
+
+### Fixed
+- Various fixes as consequence of changing file_location / file_name ==> bucket_name / object_name
+- Celery Routing / Queuing updated
+
+## [1.0.10-alfa]
+
+### Added
+- BusinessEventLog monitoring using Langchain native code
+
+### Changed
+- Allow longer audio files (or video) to be uploaded and processed
+- Storage and Embedding usage now expressed in MiB iso tokens (more logical)
+- Views for License / LicenseTier
+
+### Removed
+- Portkey removed for monitoring usage
+
+## [1.0.9-alfa] - 2024/10/01
+
+### Added
+- Business Event tracing (eveai_workers & eveai_chat_workers)
+- Flower Container added for monitoring
+
+### Changed
+- Healthcheck improvements
+- model_utils turned into a class with lazy loading
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- Set default language when registering Documents or URLs.
+
 ## [1.0.8-alfa] - 2024-09-12
 
 ### Added

Evie Overview.md

@@ -0,0 +1,85 @@
+# Evie Overview
+
+Owner: pieter Laroy
+
+# Introduction
+
+The Evie project (developed by AskEveAI) is a SAAS product that enables SMEs to easily introduce AI optimisations for both internal and external use. There are two big concepts:
+
+- Catalogs: these allow tenants to store information about their organisations or enterprises
+- Specialists: these allow tenants to perform logic into their processes, communications, …
+
+As such, we could say we have an advanced RAG system tenants can use to optimise their workings.
+
+## Multi-tenant
+
+The application has a multi-tenant setup built in. This is reflected in:
+
+- The Database:
+    - We have 1 public schema, in which general information is defined such as tenants, their users, domains, licenses, …
+    - We have a schema (named 1, 2, …) for each of the tenants defined in the system, containing all information on the tenant’s catalogs & documents, specialists & interactions, …
+- File Storage
+    - We use S3-compatible storage
+    - A bucket is defined for each tenant, storing their specific documents, assets, …
+
+That way, general information required for the operation of Evie is stored in the public schema, and specific and potentially sensitive information is nicely stored behind a Chinese wall for each of the tenants.
+
+## Partners
+
+We started to define the concept of a partner. This allows us to have partners that introduce tenants to Evie, or offer them additional functionality (specialists) or knowledge (catalogs). This concept is in an early stage at this point.
+
+## Domains
+
+In order to ensure a structured approach, we have defined several domains in the project:
+
+- **User**: the user domain is used to store all data on partners, tenants, actual users.
+- **Document**: the document domain is used to store all information on catalogs, documents, how to process documents, …
+- **Interaction**: This domain allows us to define specialists, agents, … and to interact with the specialists and agents.
+- **Entitlements**: This domain defines all license information, usage, …
+
+# Project Structure
+
+## Common
+
+The common folder contains code that is used in different components of the system. It contains the following important pieces:
+
+- **models**: in the models folder you can find the SQLAlchemy models used throughout the application. These models are organised in their relevant domains.
+- **eveai_model**: some classes to handle usage, wrappers around standard LLM clients
+- **langchain**: similar to eveai_model, but in the langchain library
+- **services**: I started to define services to define reusable functionality in the system. There again are defined in their respective domains
+- **utils**: a whole bunch of utility classes. Some should get converted to services classes in the future
+- **utils/cache**: contains code for caching different elements in the application
+
+## config
+
+The config folder contains quite some configuration data (as the name suggests):
+
+- **config.py**: general configuration
+- **logging_config.py**: definition of logging files
+- **model_config.py**: obsolete
+- **type_defs**: contains the lists of definitions for several types used throughout the application. E.g. processor_types, specialist_types, …
+- **All other folders**: detailed configuration of all the types defined in type_defs.
+
+## docker
+
+The docker folder contains the configuration and scripts used for all operations on configuring and building containers, distributing containers, …
+
+## eveai_… folders
+
+These are different components (containerized) of our application:
+
+- **eveai_api**: The API of our application.
+- **eveai_app**: The administrative interface of our application.
+- **eveai_beat**: a means to install batch processes for our application.
+- **eveai_chat**: obsolete at this moment
+- **eveai_chat_workers**: celery based invocation of our specialists
+- **eveai_client**: newly added. A desktop client to invoking specialists.
+- **eveai_entitlements**: celery based approach to handling business events, measuring and updating usage, …
+- **eveai_workers**: celery based approach to filling catalogs with documents (embedding)
+
+## Remaining folders
+
+- **integrations**: integrations to e.g. Wordpress and Zapier.
+- **migrations**: SQLAlchemy database migration files (for public and tenant schema)
+- **nginx**: configuration and static files for nginx
+- **scripts**: various scripts used to start up components, to perform database operations, …

common/eveai_model/eveai_embedding_base.py

@@ -0,0 +1,11 @@
+from abc import abstractmethod
+from typing import List
+
+
+class EveAIEmbeddings:
+    @abstractmethod
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        raise NotImplementedError
+
+    def embed_query(self, text: str) -> List[float]:
+        return self.embed_documents([text])[0]

common/eveai_model/tracked_mistral_embeddings.py

@@ -0,0 +1,141 @@
+from flask import current_app
+from langchain_mistralai import MistralAIEmbeddings
+from typing import List, Any
+import time
+
+from common.eveai_model.eveai_embedding_base import EveAIEmbeddings
+from common.utils.business_event_context import current_event
+from mistralai import Mistral
+
+
+class TrackedMistralAIEmbeddings(EveAIEmbeddings):
+    def __init__(self, model: str = "mistral_embed", batch_size: int = 10):
+        """
+        Initialize the TrackedMistralAIEmbeddings class.
+
+        Args:
+            model: The embedding model to use
+            batch_size: Maximum number of texts to send in a single API call
+        """
+        api_key = current_app.config['MISTRAL_API_KEY']
+        self.client = Mistral(
+            api_key=api_key
+        )
+        self.model = model
+        self.batch_size = batch_size
+        super().__init__()
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        """
+        Embed a list of texts, processing in batches to avoid API limitations.
+
+        Args:
+            texts: A list of texts to embed
+
+        Returns:
+            A list of embeddings, one for each input text
+        """
+        if not texts:
+            return []
+
+        all_embeddings = []
+
+        # Process texts in batches
+        for i in range(0, len(texts), self.batch_size):
+            batch = texts[i:i + self.batch_size]
+            batch_num = i // self.batch_size + 1
+            current_app.logger.debug(f"Processing embedding batch {batch_num}, size: {len(batch)}")
+
+            start_time = time.time()
+            try:
+                result = self.client.embeddings.create(
+                    model=self.model,
+                    inputs=batch
+                )
+                end_time = time.time()
+                batch_time = end_time - start_time
+
+                batch_embeddings = [embedding.embedding for embedding in result.data]
+                all_embeddings.extend(batch_embeddings)
+
+                # Log metrics for this batch
+                metrics = {
+                    'total_tokens': result.usage.total_tokens,
+                    'prompt_tokens': result.usage.prompt_tokens,
+                    'completion_tokens': result.usage.completion_tokens,
+                    'time_elapsed': batch_time,
+                    'interaction_type': 'Embedding',
+                    'batch': batch_num,
+                    'batch_size': len(batch)
+                }
+                current_event.log_llm_metrics(metrics)
+
+                current_app.logger.debug(f"Batch {batch_num} processed: {len(batch)} texts, "
+                                         f"{result.usage.total_tokens} tokens, {batch_time:.2f}s")
+
+                # If processing multiple batches, add a small delay to avoid rate limits
+                if len(texts) > self.batch_size and i + self.batch_size < len(texts):
+                    time.sleep(0.25)  # 250ms pause between batches
+
+            except Exception as e:
+                current_app.logger.error(f"Error in embedding batch {batch_num}: {str(e)}")
+                # If a batch fails, try to process each text individually
+                for j, text in enumerate(batch):
+                    try:
+                        current_app.logger.debug(f"Attempting individual embedding for item {i + j}")
+                        single_start_time = time.time()
+                        single_result = self.client.embeddings.create(
+                            model=self.model,
+                            inputs=[text]
+                        )
+                        single_end_time = time.time()
+
+                        # Add the single embedding
+                        single_embedding = single_result.data[0].embedding
+                        all_embeddings.append(single_embedding)
+
+                        # Log metrics for this individual embedding
+                        single_metrics = {
+                            'total_tokens': single_result.usage.total_tokens,
+                            'prompt_tokens': single_result.usage.prompt_tokens,
+                            'completion_tokens': single_result.usage.completion_tokens,
+                            'time_elapsed': single_end_time - single_start_time,
+                            'interaction_type': 'Embedding',
+                            'batch': f"{batch_num}-recovery-{j}",
+                            'batch_size': 1
+                        }
+                        current_event.log_llm_metrics(single_metrics)
+
+                    except Exception as inner_e:
+                        current_app.logger.error(f"Failed to embed individual text at index {i + j}: {str(inner_e)}")
+                        # Add a zero vector as a placeholder for failed embeddings
+                        # Use the correct dimensionality for the model (1024 for mistral_embed)
+                        embedding_dim = 1024
+                        all_embeddings.append([0.0] * embedding_dim)
+
+        total_batches = (len(texts) + self.batch_size - 1) // self.batch_size
+        current_app.logger.info(f"Embedded {len(texts)} texts in {total_batches} batches")
+
+        return all_embeddings
+
+    # def embed_documents(self, texts: list[str]) -> list[list[float]]:
+    #     start_time = time.time()
+    #     result = self.client.embeddings.create(
+    #         model=self.model,
+    #         inputs=texts
+    #     )
+    #     end_time = time.time()
+    #
+    #     metrics = {
+    #         'total_tokens': result.usage.total_tokens,
+    #         'prompt_tokens': result.usage.prompt_tokens,  # For embeddings, all tokens are prompt tokens
+    #         'completion_tokens': result.usage.completion_tokens,
+    #         'time_elapsed': end_time - start_time,
+    #         'interaction_type': 'Embedding',
+    #     }
+    #     current_event.log_llm_metrics(metrics)
+    #
+    #     embeddings = [embedding.embedding for embedding in result.data]
+    #
+    #     return embeddings
+

common/eveai_model/tracked_mistral_ocr_client.py

@@ -0,0 +1,53 @@
+import re
+import time
+
+from flask import current_app
+from mistralai import Mistral
+
+from common.utils.business_event_context import current_event
+
+
+class TrackedMistralOcrClient:
+    def __init__(self):
+        api_key = current_app.config['MISTRAL_API_KEY']
+        self.client = Mistral(
+            api_key=api_key,
+        )
+        self.model = "mistral-ocr-latest"
+
+    def _get_title(self, markdown):
+        # Look for the first level-1 heading
+        match = re.search(r'^# (.+)', markdown, re.MULTILINE)
+        return match.group(1).strip() if match else None
+
+    def process_pdf(self, file_name, file_content):
+        start_time = time.time()
+        uploaded_pdf = self.client.files.upload(
+            file={
+                "file_name": file_name,
+                "content": file_content
+            },
+            purpose="ocr"
+        )
+        signed_url = self.client.files.get_signed_url(file_id=uploaded_pdf.id)
+        ocr_response = self.client.ocr.process(
+            model=self.model,
+            document={
+                "type": "document_url",
+                "document_url": signed_url.url
+            },
+            include_image_base64=False
+        )
+        nr_of_pages = len(ocr_response.pages)
+        all_markdown = " ".join(page.markdown for page in ocr_response.pages)
+        title = self._get_title(all_markdown)
+        end_time = time.time()
+
+        metrics = {
+            'nr_of_pages': nr_of_pages,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'OCR',
+        }
+        current_event.log_llm_metrics(metrics)
+
+        return all_markdown, title

common/extensions.py

@@ -2,16 +2,16 @@ from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
 from flask_bootstrap import Bootstrap
 from flask_security import Security
-from flask_mailman import Mail
 from flask_login import LoginManager
 from flask_cors import CORS
-from flask_socketio import SocketIO
 from flask_jwt_extended import JWTManager
 from flask_session import Session
 from flask_wtf import CSRFProtect
 from flask_restx import Api
 from prometheus_flask_exporter import PrometheusMetrics
 
+from .utils.cache.eveai_cache_manager import EveAICacheManager
+from .utils.content_utils import ContentManager
 from .utils.simple_encryption import SimpleEncryption
 from .utils.minio_utils import MinioClient
 
@@ -22,13 +22,14 @@ migrate = Migrate()
 bootstrap = Bootstrap()
 csrf = CSRFProtect()
 security = Security()
-mail = Mail()
 login_manager = LoginManager()
 cors = CORS()
-socketio = SocketIO()
 jwt = JWTManager()
 session = Session()
 api_rest = Api()
 simple_encryption = SimpleEncryption()
 minio_client = MinioClient()
 metrics = PrometheusMetrics.for_app_factory()
+cache_manager = EveAICacheManager()
+content_manager = ContentManager()
+

common/langchain/eveai_history_retriever.py

@@ -1,52 +0,0 @@
-from langchain_core.retrievers import BaseRetriever
-from sqlalchemy import asc
-from sqlalchemy.exc import SQLAlchemyError
-from pydantic import Field, BaseModel, PrivateAttr
-from typing import Any, Dict
-from flask import current_app
-
-from common.extensions import db
-from common.models.interaction import ChatSession, Interaction
-from common.utils.model_utils import ModelVariables
-
-
-class EveAIHistoryRetriever(BaseRetriever, BaseModel):
-    _model_variables: ModelVariables = PrivateAttr()
-    _session_id: str = PrivateAttr()
-
-    def __init__(self, model_variables: ModelVariables, session_id: str):
-        super().__init__()
-        self._model_variables = model_variables
-        self._session_id = session_id
-
-    @property
-    def model_variables(self) -> ModelVariables:
-        return self._model_variables
-
-    @property
-    def session_id(self) -> str:
-        return self._session_id
-
-    def _get_relevant_documents(self, query: str):
-        current_app.logger.debug(f'Retrieving history of interactions for query: {query}')
-
-        try:
-            query_obj = (
-                db.session.query(Interaction)
-                .join(ChatSession, Interaction.chat_session_id == ChatSession.id)
-                .filter(ChatSession.session_id == self.session_id)
-                .order_by(asc(Interaction.id))
-            )
-
-            interactions = query_obj.all()
-
-            result = []
-            for interaction in interactions:
-                result.append(f'HUMAN:\n{interaction.detailed_question}\n\nAI: \n{interaction.answer}\n\n')
-
-        except SQLAlchemyError as e:
-            current_app.logger.error(f'Error retrieving history of interactions: {e}')
-            db.session.rollback()
-            return []
-
-        return result

common/langchain/eveai_retriever.py

@@ -1,138 +0,0 @@
-from langchain_core.retrievers import BaseRetriever
-from sqlalchemy import func, and_, or_, desc
-from sqlalchemy.exc import SQLAlchemyError
-from pydantic import BaseModel, Field, PrivateAttr
-from typing import Any, Dict
-from flask import current_app
-
-from common.extensions import db
-from common.models.document import Document, DocumentVersion
-from common.utils.datetime_utils import get_date_in_timezone
-from common.utils.model_utils import ModelVariables
-
-
-class EveAIRetriever(BaseRetriever, BaseModel):
-    _model_variables: ModelVariables = PrivateAttr()
-    _tenant_info: Dict[str, Any] = PrivateAttr()
-
-    def __init__(self, model_variables: ModelVariables, tenant_info: Dict[str, Any]):
-        super().__init__()
-        current_app.logger.debug(f'Model variables type: {type(model_variables)}')
-        self._model_variables = model_variables
-        self._tenant_info = tenant_info
-
-    @property
-    def model_variables(self) -> ModelVariables:
-        return self._model_variables
-
-    @property
-    def tenant_info(self) -> Dict[str, Any]:
-        return self._tenant_info
-
-    def _get_relevant_documents(self, query: str):
-        current_app.logger.debug(f'Retrieving relevant documents for query: {query}')
-        query_embedding = self._get_query_embedding(query)
-        current_app.logger.debug(f'Model Variables Private: {type(self._model_variables)}')
-        current_app.logger.debug(f'Model Variables Property: {type(self.model_variables)}')
-        db_class = self.model_variables['embedding_db_model']
-        similarity_threshold = self.model_variables['similarity_threshold']
-        k = self.model_variables['k']
-
-        if self.tenant_info['rag_tuning']:
-            try:
-                current_date = get_date_in_timezone(self.tenant_info['timezone'])
-                current_app.rag_tuning_logger.debug(f'Current date: {current_date}\n')
-
-                # Debug query to show similarity for all valid documents (without chunk text)
-                debug_query = (
-                    db.session.query(
-                        Document.id.label('document_id'),
-                        DocumentVersion.id.label('version_id'),
-                        db_class.id.label('embedding_id'),
-                        (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity')
-                    )
-                    .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
-                    .join(Document, DocumentVersion.doc_id == Document.id)
-                    .filter(
-                        or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
-                        or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date)
-                    )
-                    .order_by(desc('similarity'))
-                )
-
-                debug_results = debug_query.all()
-
-                current_app.logger.debug("Debug: Similarity for all valid documents:")
-                for row in debug_results:
-                    current_app.rag_tuning_logger.debug(f"Doc ID: {row.document_id}, "
-                                                        f"Version ID: {row.version_id}, "
-                                                        f"Embedding ID: {row.embedding_id}, "
-                                                        f"Similarity: {row.similarity}")
-                current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
-            except SQLAlchemyError as e:
-                current_app.logger.error(f'Error generating overview: {e}')
-                db.session.rollback()
-
-        if self.tenant_info['rag_tuning']:
-            current_app.rag_tuning_logger.debug(f'Parameters for Retrieval of documents: \n')
-            current_app.rag_tuning_logger.debug(f'Similarity Threshold: {similarity_threshold}\n')
-            current_app.rag_tuning_logger.debug(f'K: {k}\n')
-            current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
-
-        try:
-            current_date = get_date_in_timezone(self.tenant_info['timezone'])
-            # Subquery to find the latest version of each document
-            subquery = (
-                db.session.query(
-                    DocumentVersion.doc_id,
-                    func.max(DocumentVersion.id).label('latest_version_id')
-                )
-                .group_by(DocumentVersion.doc_id)
-                .subquery()
-            )
-            # Main query to filter embeddings
-            query_obj = (
-                db.session.query(db_class,
-                                 (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity'))
-                .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
-                .join(Document, DocumentVersion.doc_id == Document.id)
-                .join(subquery, DocumentVersion.id == subquery.c.latest_version_id)
-                .filter(
-                    or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
-                    or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date),
-                    (1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold
-                )
-                .order_by(desc('similarity'))
-                .limit(k)
-            )
-
-            if self.tenant_info['rag_tuning']:
-                current_app.rag_tuning_logger.debug(f'Query executed for Retrieval of documents: \n')
-                current_app.rag_tuning_logger.debug(f'{query_obj.statement}\n')
-                current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
-
-            res = query_obj.all()
-
-            if self.tenant_info['rag_tuning']:
-                current_app.rag_tuning_logger.debug(f'Retrieved {len(res)} relevant documents \n')
-                current_app.rag_tuning_logger.debug(f'Data retrieved: \n')
-                current_app.rag_tuning_logger.debug(f'{res}\n')
-                current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
-
-            result = []
-            for doc in res:
-                if self.tenant_info['rag_tuning']:
-                    current_app.rag_tuning_logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n')
-                    current_app.rag_tuning_logger.debug(f'Chunk: \n {doc[0].chunk}\n\n')
-                result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n')
-
-        except SQLAlchemyError as e:
-            current_app.logger.error(f'Error retrieving relevant documents: {e}')
-            db.session.rollback()
-            return []
-        return result
-
-    def _get_query_embedding(self, query: str):
-        embedding_model = self.model_variables['embedding_model']
-        query_embedding = embedding_model.embed_query(query)
-        return query_embedding

common/langchain/llm_metrics_handler.py

@@ -0,0 +1,49 @@
+import time
+from langchain.callbacks.base import BaseCallbackHandler
+from typing import Dict, Any, List
+from langchain.schema import LLMResult
+from common.utils.business_event_context import current_event
+from flask import current_app
+
+
+class LLMMetricsHandler(BaseCallbackHandler):
+    def __init__(self):
+        self.total_tokens: int = 0
+        self.prompt_tokens: int = 0
+        self.completion_tokens: int = 0
+        self.start_time: float = 0
+        self.end_time: float = 0
+        self.total_time: float = 0
+
+    def reset(self):
+        self.total_tokens = 0
+        self.prompt_tokens = 0
+        self.completion_tokens = 0
+        self.start_time = 0
+        self.end_time = 0
+        self.total_time = 0
+
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        self.start_time = time.time()
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        self.end_time = time.time()
+        self.total_time = self.end_time - self.start_time
+
+        usage = response.llm_output.get('token_usage', {})
+        self.prompt_tokens += usage.get('prompt_tokens', 0)
+        self.completion_tokens += usage.get('completion_tokens', 0)
+        self.total_tokens = self.prompt_tokens + self.completion_tokens
+
+        metrics = self.get_metrics()
+        current_event.log_llm_metrics(metrics)
+        self.reset()  # Reset for the next call
+
+    def get_metrics(self) -> Dict[str, int | float]:
+        return {
+            'total_tokens': self.total_tokens,
+            'prompt_tokens': self.prompt_tokens,
+            'completion_tokens': self.completion_tokens,
+            'time_elapsed': self.total_time,
+            'interaction_type': 'LLM',
+        }

common/langchain/outputs/base.py

@@ -0,0 +1,23 @@
+# Output Schema Management - common/langchain/outputs/base.py
+from typing import Dict, Type, Any
+from pydantic import BaseModel
+
+
+class BaseSpecialistOutput(BaseModel):
+    """Base class for all specialist outputs"""
+    pass
+
+
+class OutputRegistry:
+    """Registry for specialist output schemas"""
+    _schemas: Dict[str, Type[BaseSpecialistOutput]] = {}
+
+    @classmethod
+    def register(cls, specialist_type: str, schema_class: Type[BaseSpecialistOutput]):
+        cls._schemas[specialist_type] = schema_class
+
+    @classmethod
+    def get_schema(cls, specialist_type: str) -> Type[BaseSpecialistOutput]:
+        if specialist_type not in cls._schemas:
+            raise ValueError(f"No output schema registered for {specialist_type}")
+        return cls._schemas[specialist_type]

common/langchain/outputs/rag.py

@@ -0,0 +1,22 @@
+# RAG Specialist Output - common/langchain/outputs/rag.py
+from typing import List
+from pydantic import Field
+from .base import BaseSpecialistOutput
+
+
+class RAGOutput(BaseSpecialistOutput):
+    """Output schema for RAG specialist"""
+    """Default docstring - to be replaced with actual prompt"""
+
+    answer: str = Field(
+        ...,
+        description="The answer to the user question, based on the given sources",
+    )
+    citations: List[int] = Field(
+        ...,
+        description="The integer IDs of the SPECIFIC sources that were used to generate the answer"
+    )
+    insufficient_info: bool = Field(
+        False,  # Default value is set to False
+        description="A boolean indicating whether given sources were sufficient or not to generate the answer"
+    )

common/langchain/tracked_openai_embeddings.py

@@ -0,0 +1,51 @@
+from langchain_openai import OpenAIEmbeddings
+from typing import List, Any
+import time
+from common.utils.business_event_context import current_event
+
+
+class TrackedOpenAIEmbeddings(OpenAIEmbeddings):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        start_time = time.time()
+        result = super().embed_documents(texts)
+        end_time = time.time()
+
+        # Estimate token usage (OpenAI uses tiktoken for this)
+        import tiktoken
+        enc = tiktoken.encoding_for_model(self.model)
+        total_tokens = sum(len(enc.encode(text)) for text in texts)
+
+        metrics = {
+            'total_tokens': total_tokens,
+            'prompt_tokens': total_tokens,  # For embeddings, all tokens are prompt tokens
+            'completion_tokens': 0,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'Embedding',
+            }
+        current_event.log_llm_metrics(metrics)
+
+        return result
+
+    def embed_query(self, text: str) -> List[float]:
+        start_time = time.time()
+        result = super().embed_query(text)
+        end_time = time.time()
+
+        # Estimate token usage
+        import tiktoken
+        enc = tiktoken.encoding_for_model(self.model)
+        total_tokens = len(enc.encode(text))
+
+        metrics = {
+            'total_tokens': total_tokens,
+            'prompt_tokens': total_tokens,
+            'completion_tokens': 0,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'Embedding',
+        }
+        current_event.log_llm_metrics(metrics)
+
+        return result

common/langchain/tracked_transcription.py

@@ -0,0 +1,77 @@
+# common/langchain/tracked_transcription.py
+from typing import Any, Optional, Dict
+import time
+from openai import OpenAI
+from common.utils.business_event_context import current_event
+
+
+class TrackedOpenAITranscription:
+    """Wrapper for OpenAI transcription with metric tracking"""
+
+    def __init__(self, api_key: str, **kwargs: Any):
+        """Initialize with OpenAI client settings"""
+        self.client = OpenAI(api_key=api_key)
+        self.model = kwargs.get('model', 'whisper-1')
+
+    def transcribe(self,
+                   file: Any,
+                   model: Optional[str] = None,
+                   language: Optional[str] = None,
+                   prompt: Optional[str] = None,
+                   response_format: Optional[str] = None,
+                   temperature: Optional[float] = None,
+                   duration: Optional[int] = None) -> str:
+        """
+        Transcribe audio with metrics tracking
+
+        Args:
+            file: Audio file to transcribe
+            model: Model to use (defaults to whisper-1)
+            language: Optional language of the audio
+            prompt: Optional prompt to guide transcription
+            response_format: Response format (json, text, etc)
+            temperature: Sampling temperature
+            duration: Duration of audio in seconds for metrics
+
+        Returns:
+            Transcription text
+        """
+        start_time = time.time()
+
+        try:
+            # Create transcription options
+            options = {
+                "file": file,
+                "model": model or self.model,
+            }
+            if language:
+                options["language"] = language
+            if prompt:
+                options["prompt"] = prompt
+            if response_format:
+                options["response_format"] = response_format
+            if temperature:
+                options["temperature"] = temperature
+
+            response = self.client.audio.transcriptions.create(**options)
+
+            # Calculate metrics
+            end_time = time.time()
+
+            # Token usage for transcriptions is based on audio duration
+            metrics = {
+                'total_tokens': duration or 600,  # Default to 10 minutes if duration not provided
+                'prompt_tokens': 0,  # For transcriptions, all tokens are completion
+                'completion_tokens': duration or 600,
+                'time_elapsed': end_time - start_time,
+                'interaction_type': 'ASR',
+            }
+            current_event.log_llm_metrics(metrics)
+
+            # Return text from response
+            if isinstance(response, str):
+                return response
+            return response.text
+
+        except Exception as e:
+            raise Exception(f"Transcription failed: {str(e)}")

common/models/document.py

@@ -2,12 +2,81 @@ from common.extensions import db
 from .user import User, Tenant
 from pgvector.sqlalchemy import Vector
 from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import ARRAY
+import sqlalchemy as sa
+
+
+class Catalog(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False, unique=True)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_CATALOG")
+
+    min_chunk_size = db.Column(db.Integer, nullable=True, default=1500)
+    max_chunk_size = db.Column(db.Integer, nullable=True, default=2500)
+
+    # Meta Data
+    user_metadata = db.Column(JSONB, nullable=True)
+    system_metadata = db.Column(JSONB, nullable=True)
+    configuration = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class Processor(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True)
+    type = db.Column(db.String(50), nullable=False)
+    sub_file_type = db.Column(db.String(50), nullable=True)
+
+    # Tuning enablers
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+
+    # Meta Data
+    user_metadata = db.Column(JSONB, nullable=True)
+    system_metadata = db.Column(JSONB, nullable=True)
+    configuration = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class Retriever(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="STANDARD_RAG")
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+
+    # Meta Data
+    user_metadata = db.Column(JSONB, nullable=True)
+    system_metadata = db.Column(JSONB, nullable=True)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
 
 class Document(db.Model):
     id = db.Column(db.Integer, primary_key=True)
+    # tenant_id = db.Column(db.Integer, db.ForeignKey(Tenant.id), nullable=False)
+    catalog_id = db.Column(db.Integer, db.ForeignKey(Catalog.id), nullable=True)
     name = db.Column(db.String(100), nullable=False)
-    tenant_id = db.Column(db.Integer, db.ForeignKey(Tenant.id), nullable=False)
     valid_from = db.Column(db.DateTime, nullable=True)
     valid_to = db.Column(db.DateTime, nullable=True)
 
@@ -28,14 +97,17 @@ class DocumentVersion(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     doc_id = db.Column(db.Integer, db.ForeignKey(Document.id), nullable=False)
     url = db.Column(db.String(200), nullable=True)
-    file_location = db.Column(db.String(255), nullable=True)
-    file_name = db.Column(db.String(200), nullable=True)
+    bucket_name = db.Column(db.String(255), nullable=True)
+    object_name = db.Column(db.String(200), nullable=True)
     file_type = db.Column(db.String(20), nullable=True)
+    sub_file_type = db.Column(db.String(50), nullable=True)
+    file_size = db.Column(db.Float, nullable=True)
     language = db.Column(db.String(2), nullable=False)
     user_context = db.Column(db.Text, nullable=True)
     system_context = db.Column(db.Text, nullable=True)
     user_metadata = db.Column(JSONB, nullable=True)
     system_metadata = db.Column(JSONB, nullable=True)
+    catalog_properties = db.Column(JSONB, nullable=True)
 
     # Versioning Information
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
@@ -55,12 +127,6 @@ class DocumentVersion(db.Model):
     def __repr__(self):
         return f"<DocumentVersion {self.document_language.document_id}.{self.document_language.language}>.{self.id}>"
 
-    def calc_file_location(self):
-        return f"{self.document.tenant_id}/{self.document.id}/{self.language}"
-
-    def calc_file_name(self):
-        return f"{self.id}.{self.file_type}"
-
 
 class Embedding(db.Model):
     __tablename__ = 'embeddings'

common/models/entitlements.py

@@ -0,0 +1,528 @@
+from sqlalchemy.sql.expression import text
+
+from common.extensions import db
+from datetime import datetime as dt, timezone as tz
+from enum import Enum
+from sqlalchemy import event
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.hybrid import hybrid_property
+from dateutil.relativedelta import relativedelta
+
+from common.utils.database import Database
+
+
+class BusinessEventLog(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    timestamp = db.Column(db.DateTime, nullable=False)
+    event_type = db.Column(db.String(50), nullable=False)
+    tenant_id = db.Column(db.Integer, nullable=False)
+    trace_id = db.Column(db.String(50), nullable=False)
+    span_id = db.Column(db.String(50))
+    span_name = db.Column(db.String(255))
+    parent_span_id = db.Column(db.String(50))
+    document_version_id = db.Column(db.Integer)
+    document_version_file_size = db.Column(db.Float)
+    specialist_id = db.Column(db.Integer)
+    specialist_type = db.Column(db.String(50))
+    specialist_type_version = db.Column(db.String(20))
+    chat_session_id = db.Column(db.String(50))
+    interaction_id = db.Column(db.Integer)
+    environment = db.Column(db.String(20))
+    llm_metrics_total_tokens = db.Column(db.Integer)
+    llm_metrics_prompt_tokens = db.Column(db.Integer)
+    llm_metrics_completion_tokens = db.Column(db.Integer)
+    llm_metrics_total_time = db.Column(db.Float)
+    llm_metrics_nr_of_pages = db.Column(db.Integer)
+    llm_metrics_call_count = db.Column(db.Integer)
+    llm_interaction_type = db.Column(db.String(20))
+    message = db.Column(db.Text)
+    license_usage_id = db.Column(db.Integer, db.ForeignKey('public.license_usage.id'), nullable=True)
+    license_usage = db.relationship('LicenseUsage', backref='events')
+
+
+class License(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    tier_id = db.Column(db.Integer, db.ForeignKey('public.license_tier.id'),nullable=False)  # 'small', 'medium', 'custom'
+    start_date = db.Column(db.Date, nullable=False)
+    end_date = db.Column(db.Date, nullable=True)
+    nr_of_periods = db.Column(db.Integer, nullable=False)
+    currency = db.Column(db.String(20), nullable=False)
+    yearly_payment = db.Column(db.Boolean, nullable=False, default=False)
+    basic_fee = db.Column(db.Float, nullable=False)
+    max_storage_mb = db.Column(db.Integer, nullable=False)
+    additional_storage_price = db.Column(db.Float, nullable=False)
+    additional_storage_bucket = db.Column(db.Integer, nullable=False)
+    included_embedding_mb = db.Column(db.Integer, nullable=False)
+    additional_embedding_price = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_embedding_bucket = db.Column(db.Integer, nullable=False)
+    included_interaction_tokens = db.Column(db.Integer, nullable=False)
+    additional_interaction_token_price = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_interaction_bucket = db.Column(db.Integer, nullable=False)
+    overage_embedding = db.Column(db.Float, nullable=False, default=0)
+    overage_interaction = db.Column(db.Float, nullable=False, default=0)
+    additional_storage_allowed = db.Column(db.Boolean, nullable=False, default=False)
+    additional_embedding_allowed = db.Column(db.Boolean, nullable=False, default=False)
+    additional_interaction_allowed = db.Column(db.Boolean, nullable=False, default=False)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    tenant = db.relationship('Tenant', back_populates='licenses')
+    license_tier = db.relationship('LicenseTier', back_populates='licenses')
+    periods = db.relationship('LicensePeriod', back_populates='license',
+                              order_by='LicensePeriod.period_number',
+                              cascade='all, delete-orphan')
+
+def calculate_end_date(start_date, nr_of_periods):
+    """Utility functie om einddatum te berekenen"""
+    if start_date and nr_of_periods:
+        return start_date + relativedelta(months=nr_of_periods) - relativedelta(days=1)
+    return None
+
+# Luister naar start_date wijzigingen
+@event.listens_for(License.start_date, 'set')
+def set_start_date(target, value, oldvalue, initiator):
+    """Bijwerken van end_date wanneer start_date wordt aangepast"""
+    if value and target.nr_of_periods:
+        target.end_date = calculate_end_date(value, target.nr_of_periods)
+
+# Luister naar nr_of_periods wijzigingen
+@event.listens_for(License.nr_of_periods, 'set')
+def set_nr_of_periods(target, value, oldvalue, initiator):
+    """Bijwerken van end_date wanneer nr_of_periods wordt aangepast"""
+    if value and target.start_date:
+        target.end_date = calculate_end_date(target.start_date, value)
+
+
+class LicenseTier(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    version = db.Column(db.String(50), nullable=False)
+    start_date = db.Column(db.Date, nullable=False)
+    end_date = db.Column(db.Date, nullable=True)
+    basic_fee_d = db.Column(db.Float, nullable=True)
+    basic_fee_e = db.Column(db.Float, nullable=True)
+    max_storage_mb = db.Column(db.Integer, nullable=False)
+    additional_storage_price_d = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_storage_price_e = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_storage_bucket = db.Column(db.Integer, nullable=False)
+    included_embedding_mb = db.Column(db.Integer, nullable=False)
+    additional_embedding_price_d = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_embedding_price_e = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_embedding_bucket = db.Column(db.Integer, nullable=False)
+    included_interaction_tokens = db.Column(db.Integer, nullable=False)
+    additional_interaction_token_price_d = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_interaction_token_price_e = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_interaction_bucket = db.Column(db.Integer, nullable=False)
+    standard_overage_embedding = db.Column(db.Float, nullable=False, default=0)
+    standard_overage_interaction = db.Column(db.Float, nullable=False, default=0)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    licenses = db.relationship('License', back_populates='license_tier')
+    partner_services = db.relationship('PartnerServiceLicenseTier', back_populates='license_tier')
+
+
+class PartnerServiceLicenseTier(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    partner_service_id = db.Column(db.Integer, db.ForeignKey('public.partner_service.id'), primary_key=True,
+                                   nullable=False)
+    license_tier_id = db.Column(db.Integer, db.ForeignKey('public.license_tier.id'), primary_key=True,
+                                nullable=False)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    license_tier = db.relationship('LicenseTier', back_populates='partner_services')
+    partner_service = db.relationship('PartnerService', back_populates='license_tiers')
+
+
+class PeriodStatus(Enum):
+    UPCOMING = "UPCOMING"   # The period is still in the future
+    PENDING = "PENDING"     # The period is active, but prepaid is not yet received
+    ACTIVE = "ACTIVE"       # The period is active and prepaid has been received
+    COMPLETED = "COMPLETED" # The period has been completed, but not yet invoiced
+    INVOICED = "INVOICED"   # The period has been completed and invoiced, but overage payment still pending
+    CLOSED = "CLOSED"       # The period has been closed, invoiced and fully paid
+
+
+class LicensePeriod(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+    # Period identification
+    period_number = db.Column(db.Integer, nullable=False)
+    period_start = db.Column(db.Date, nullable=False)
+    period_end = db.Column(db.Date, nullable=False)
+
+    # License configuration snapshot - copied from license when period is created
+    currency = db.Column(db.String(20), nullable=True)
+    basic_fee = db.Column(db.Float, nullable=True)
+    max_storage_mb = db.Column(db.Integer, nullable=True)
+    additional_storage_price = db.Column(db.Float, nullable=True)
+    additional_storage_bucket = db.Column(db.Integer, nullable=True)
+    included_embedding_mb = db.Column(db.Integer, nullable=True)
+    additional_embedding_price = db.Column(db.Numeric(10, 4), nullable=True)
+    additional_embedding_bucket = db.Column(db.Integer, nullable=True)
+    included_interaction_tokens = db.Column(db.Integer, nullable=True)
+    additional_interaction_token_price = db.Column(db.Numeric(10, 4), nullable=True)
+    additional_interaction_bucket = db.Column(db.Integer, nullable=True)
+    
+    # Allowance flags - can be changed from False to True within a period
+    additional_storage_allowed = db.Column(db.Boolean, nullable=True, default=False)
+    additional_embedding_allowed = db.Column(db.Boolean, nullable=True, default=False)
+    additional_interaction_allowed = db.Column(db.Boolean, nullable=True, default=False)
+
+    # Status tracking
+    status = db.Column(db.Enum(PeriodStatus), nullable=False, default=PeriodStatus.UPCOMING)
+
+    # State transition timestamps
+    upcoming_at = db.Column(db.DateTime, nullable=True)
+    pending_at = db.Column(db.DateTime, nullable=True)
+    active_at = db.Column(db.DateTime, nullable=True)
+    completed_at = db.Column(db.DateTime, nullable=True)
+    invoiced_at = db.Column(db.DateTime, nullable=True)
+    closed_at = db.Column(db.DateTime, nullable=True)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relationships
+    license = db.relationship('License', back_populates='periods')
+    license_usage = db.relationship('LicenseUsage',
+                                    uselist=False,  # This makes it one-to-one
+                                    back_populates='license_period',
+                                    cascade='all, delete-orphan')
+    payments = db.relationship('Payment', back_populates='license_period')
+    invoices = db.relationship('Invoice', back_populates='license_period',
+                               cascade='all, delete-orphan')
+
+    def update_allowance(self, allowance_type, allow_value, user_id=None):
+        """
+        Update an allowance flag within a period
+        Only allows transitioning from False to True
+        
+        Args:
+            allowance_type: One of 'storage', 'embedding', or 'interaction'
+            allow_value: The new value (must be True)
+            user_id: User ID performing the update
+            
+        Raises:
+            ValueError: If trying to change from True to False, or invalid allowance type
+        """
+        field_name = f"additional_{allowance_type}_allowed"
+        
+        # Verify valid field
+        if not hasattr(self, field_name):
+            raise ValueError(f"Invalid allowance type: {allowance_type}")
+            
+        # Get current value
+        current_value = getattr(self, field_name)
+        
+        # Only allow False -> True transition
+        if current_value is True and allow_value is True:
+            # Already True, no change needed
+            return
+        elif allow_value is False:
+            raise ValueError(f"Cannot change {field_name} from {current_value} to False")
+        
+        # Update the field
+        setattr(self, field_name, True)
+        self.updated_at = dt.now(tz.utc)
+        if user_id:
+            self.updated_by = user_id
+
+    @property
+    def prepaid_invoice(self):
+        """Get the prepaid invoice for this period"""
+        return Invoice.query.filter_by(
+            license_period_id=self.id,
+            invoice_type=PaymentType.PREPAID
+        ).first()
+
+    @property
+    def overage_invoice(self):
+        """Get the overage invoice for this period"""
+        return Invoice.query.filter_by(
+            license_period_id=self.id,
+            invoice_type=PaymentType.POSTPAID
+        ).first()
+
+    @property
+    def prepaid_payment(self):
+        """Get the prepaid payment for this period"""
+        return Payment.query.filter_by(
+            license_period_id=self.id,
+            payment_type=PaymentType.PREPAID
+        ).first()
+
+    @property
+    def overage_payment(self):
+        """Get the overage payment for this period"""
+        return Payment.query.filter_by(
+            license_period_id=self.id,
+            payment_type=PaymentType.POSTPAID
+        ).first()
+
+    @property
+    def all_invoices(self):
+        """Get all invoices for this period"""
+        return self.invoices
+
+    @property
+    def all_payments(self):
+        """Get all payments for this period"""
+        return self.payments
+
+    def transition_status(self, new_status: PeriodStatus, user_id: int = None):
+        """Transition to a new status with proper validation and logging"""
+        if not self.can_transition_to(new_status):
+            raise ValueError(f"Invalid status transition from {self.status} to {new_status}")
+
+        self.status = new_status
+        self.updated_at = dt.now(tz.utc)
+        if user_id:
+            self.updated_by = user_id
+
+        # Set appropriate timestamps
+        if new_status == PeriodStatus.ACTIVE and not self.prepaid_received_at:
+            self.prepaid_received_at = dt.now(tz.utc)
+        elif new_status == PeriodStatus.COMPLETED:
+            self.completed_at = dt.now(tz.utc)
+        elif new_status == PeriodStatus.INVOICED:
+            self.invoiced_at = dt.now(tz.utc)
+        elif new_status == PeriodStatus.CLOSED:
+            self.closed_at = dt.now(tz.utc)
+
+    @property
+    def is_overdue(self):
+        """Check if a prepaid payment is overdue"""
+        return (self.status == PeriodStatus.PENDING and
+                self.period_start <= dt.now(tz.utc).date())
+
+    def can_transition_to(self, new_status: PeriodStatus) -> bool:
+        """Check if a status transition is valid"""
+        valid_transitions = {
+            PeriodStatus.UPCOMING: [PeriodStatus.ACTIVE, PeriodStatus.PENDING],
+            PeriodStatus.PENDING: [PeriodStatus.ACTIVE],
+            PeriodStatus.ACTIVE: [PeriodStatus.COMPLETED],
+            PeriodStatus.COMPLETED: [PeriodStatus.INVOICED, PeriodStatus.CLOSED],
+            PeriodStatus.INVOICED: [PeriodStatus.CLOSED],
+            PeriodStatus.CLOSED: []
+        }
+        return new_status in valid_transitions.get(self.status, [])
+
+    def __repr__(self):
+        return f'<LicensePeriod {self.id}: License {self.license_id}, Period {self.period_number}>'
+
+
+class LicenseUsage(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    storage_mb_used = db.Column(db.Float, default=0)
+    embedding_mb_used = db.Column(db.Float, default=0)
+    embedding_prompt_tokens_used = db.Column(db.Integer, default=0)
+    embedding_completion_tokens_used = db.Column(db.Integer, default=0)
+    embedding_total_tokens_used = db.Column(db.Integer, default=0)
+    interaction_prompt_tokens_used = db.Column(db.Integer, default=0)
+    interaction_completion_tokens_used = db.Column(db.Integer, default=0)
+    interaction_total_tokens_used = db.Column(db.Integer, default=0)
+    license_period_id = db.Column(db.Integer, db.ForeignKey('public.license_period.id'), nullable=False)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    license_period = db.relationship('LicensePeriod', back_populates='license_usage')
+
+    def recalculate_storage(self):
+        Database(self.tenant_id).switch_schema()
+        # Perform a SUM operation to get the total file size from document_versions
+        total_storage = db.session.execute(text(f"""
+                SELECT SUM(file_size) 
+                FROM document_version
+            """)).scalar()
+
+        self.storage_mb_used = total_storage
+
+
+class PaymentType(Enum):
+    PREPAID = "PREPAID"
+    POSTPAID = "POSTPAID"
+
+
+class PaymentStatus(Enum):
+    PENDING = "PENDING"
+    PAID = "PAID"
+    FAILED = "FAILED"
+    CANCELLED = "CANCELLED"
+
+
+class Payment(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_period_id = db.Column(db.Integer, db.ForeignKey('public.license_period.id'), nullable=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+    # Payment details
+    payment_type = db.Column(db.Enum(PaymentType), nullable=False)
+    amount = db.Column(db.Numeric(10, 2), nullable=False)
+    currency = db.Column(db.String(3), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+
+    # Status tracking
+    status = db.Column(db.Enum(PaymentStatus), nullable=False, default=PaymentStatus.PENDING)
+
+    # External provider information
+    external_payment_id = db.Column(db.String(255), nullable=True)
+    payment_method = db.Column(db.String(50), nullable=True)  # credit_card, bank_transfer, etc.
+    provider_data = db.Column(JSONB, nullable=True)  # Provider-specific data
+
+    # Payment information
+    paid_at = db.Column(db.DateTime, nullable=True)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relationships
+    license_period = db.relationship('LicensePeriod', back_populates='payments')
+    invoice = db.relationship('Invoice', back_populates='payment', uselist=False)
+
+    @property
+    def is_overdue(self):
+        """Check if payment is overdue"""
+        if self.status != PaymentStatus.PENDING:
+            return False
+
+        # For prepaid payments, check if period start has passed
+        if (self.payment_type == PaymentType.PREPAID and
+                self.license_period_id):
+            return self.license_period.period_start <= dt.now(tz.utc).date()
+
+        # For postpaid, check against due date (would be on invoice)
+        return False
+
+    def __repr__(self):
+        return f'<Payment {self.id}: {self.payment_type} {self.amount} {self.currency}>'
+
+
+class InvoiceStatus(Enum):
+    DRAFT = "DRAFT"
+    SENT = "SENT"
+    PAID = "PAID"
+    OVERDUE = "OVERDUE"
+    CANCELLED = "CANCELLED"
+
+
+class Invoice(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_period_id = db.Column(db.Integer, db.ForeignKey('public.license_period.id'), nullable=False)
+    payment_id = db.Column(db.Integer, db.ForeignKey('public.payment.id'), nullable=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+    # Invoice details
+    invoice_type = db.Column(db.Enum(PaymentType), nullable=False)
+    invoice_number = db.Column(db.String(50), unique=True, nullable=False)
+    invoice_date = db.Column(db.Date, nullable=False)
+    due_date = db.Column(db.Date, nullable=False)
+
+    # Financial details
+    amount = db.Column(db.Numeric(10, 2), nullable=False)
+    currency = db.Column(db.String(3), nullable=False)
+    tax_amount = db.Column(db.Numeric(10, 2), default=0)
+
+    # Descriptive fields
+    description = db.Column(db.Text, nullable=True)
+    status = db.Column(db.Enum(InvoiceStatus), nullable=False, default=InvoiceStatus.DRAFT)
+
+    # Timestamps
+    sent_at = db.Column(db.DateTime, nullable=True)
+    paid_at = db.Column(db.DateTime, nullable=True)
+
+    # Standard audit fields
+    created_at = db.Column(db.DateTime, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+    updated_at = db.Column(db.DateTime, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relationships
+    license_period = db.relationship('LicensePeriod', back_populates='invoices')
+    payment = db.relationship('Payment', back_populates='invoice')
+
+    def __repr__(self):
+        return f'<Invoice {self.invoice_number}: {self.amount} {self.currency}>'
+
+
+class LicenseChangeLog(db.Model):
+    """
+    Log of changes to license configurations
+    Used for auditing and tracking when/why license details changed
+    """
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+    
+    id = db.Column(db.Integer, primary_key=True)
+    license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
+    changed_at = db.Column(db.DateTime, nullable=False, default=lambda: dt.now(tz.utc))
+    
+    # What changed
+    field_name = db.Column(db.String(100), nullable=False)
+    old_value = db.Column(db.String(255), nullable=True)
+    new_value = db.Column(db.String(255), nullable=False)
+    
+    # Why it changed
+    reason = db.Column(db.Text, nullable=True)
+    
+    # Standard audit fields
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    
+    # Relationships
+    license = db.relationship('License', backref=db.backref('change_logs', order_by='LicenseChangeLog.changed_at'))
+    
+    def __repr__(self):
+        return f'<LicenseChangeLog: {self.license_id} {self.field_name} {self.old_value} -> {self.new_value}>'
+
+
+

common/models/interaction.py

@@ -1,12 +1,14 @@
+from sqlalchemy.dialects.postgresql import JSONB
+
 from ..extensions import db
 from .user import User, Tenant
-from .document import Embedding
+from .document import Embedding, Retriever
 
 
 class ChatSession(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     user_id = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
-    session_id = db.Column(db.String(36), nullable=True)
+    session_id = db.Column(db.String(49), nullable=True)
     session_start = db.Column(db.DateTime, nullable=False)
     session_end = db.Column(db.DateTime, nullable=True)
     timezone = db.Column(db.String(30), nullable=True)
@@ -18,14 +20,168 @@ class ChatSession(db.Model):
         return f"<ChatSession {self.id} by {self.user_id}>"
 
 
+class Specialist(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Relationship to retrievers through the association table
+    retrievers = db.relationship('SpecialistRetriever', backref='specialist', lazy=True,
+                                 cascade="all, delete-orphan")
+    agents = db.relationship('EveAIAgent', backref='specialist', lazy=True)
+    tasks = db.relationship('EveAITask', backref='specialist', lazy=True)
+    tools = db.relationship('EveAITool', backref='specialist', lazy=True)
+    dispatchers = db.relationship('SpecialistDispatcher', backref='specialist', lazy=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class EveAIAsset(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="DOCUMENT_TEMPLATE")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    valid_from = db.Column(db.DateTime, nullable=True)
+    valid_to = db.Column(db.DateTime, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+    # Relations
+    versions = db.relationship('EveAIAssetVersion', backref='asset', lazy=True)
+
+
+class EveAIAssetVersion(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    asset_id = db.Column(db.Integer, db.ForeignKey(EveAIAsset.id), nullable=False)
+    bucket_name = db.Column(db.String(255), nullable=True)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+    # Relations
+    instructions = db.relationship('EveAIAssetInstruction', backref='asset_version', lazy=True)
+
+
+class EveAIAssetInstruction(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    asset_version_id = db.Column(db.Integer, db.ForeignKey(EveAIAssetVersion.id), nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    content = db.Column(db.Text, nullable=True)
+
+
+class EveAIProcessedAsset(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    asset_version_id = db.Column(db.Integer, db.ForeignKey(EveAIAssetVersion.id), nullable=False)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=True)
+    chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=True)
+    bucket_name = db.Column(db.String(255), nullable=True)
+    object_name = db.Column(db.String(255), nullable=True)
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+
+
+class EveAIAgent(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    role = db.Column(db.Text, nullable=True)
+    goal = db.Column(db.Text, nullable=True)
+    backstory = db.Column(db.Text, nullable=True)
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class EveAITask(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    task_description = db.Column(db.Text, nullable=True)
+    expected_output = db.Column(db.Text, nullable=True)
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+    context = db.Column(JSONB, nullable=True)
+    asynchronous = db.Column(db.Boolean, nullable=True, default=False)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class EveAITool(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class Dispatcher(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
 class Interaction(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=False)
-    question = db.Column(db.Text, nullable=False)
-    detailed_question = db.Column(db.Text, nullable=True)
-    answer = db.Column(db.Text, nullable=True)
-    algorithm_used = db.Column(db.String(20), nullable=True)
-    language = db.Column(db.String(2), nullable=False)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=True)
+    specialist_arguments = db.Column(JSONB, nullable=True)
+    specialist_results = db.Column(JSONB, nullable=True)
     timezone = db.Column(db.String(30), nullable=True)
     appreciation = db.Column(db.Integer, nullable=True)
 
@@ -33,6 +189,7 @@ class Interaction(db.Model):
     question_at = db.Column(db.DateTime, nullable=False)
     detailed_question_at = db.Column(db.DateTime, nullable=True)
     answer_at = db.Column(db.DateTime, nullable=True)
+    processing_error = db.Column(db.String(255), nullable=True)
 
     # Relations
     embeddings = db.relationship('InteractionEmbedding', backref='interaction', lazy=True)
@@ -44,3 +201,38 @@ class Interaction(db.Model):
 class InteractionEmbedding(db.Model):
     interaction_id = db.Column(db.Integer, db.ForeignKey(Interaction.id, ondelete='CASCADE'), primary_key=True)
     embedding_id = db.Column(db.Integer, db.ForeignKey(Embedding.id, ondelete='CASCADE'), primary_key=True)
+
+
+class SpecialistRetriever(db.Model):
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True)
+    retriever_id = db.Column(db.Integer, db.ForeignKey(Retriever.id, ondelete='CASCADE'), primary_key=True)
+
+    retriever = db.relationship("Retriever", backref="specialist_retrievers")
+
+
+class SpecialistDispatcher(db.Model):
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True)
+    dispatcher_id = db.Column(db.Integer, db.ForeignKey(Dispatcher.id, ondelete='CASCADE'), primary_key=True)
+
+    dispatcher = db.relationship("Dispatcher", backref="specialist_dispatchers")
+
+
+class SpecialistMagicLink(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), nullable=False)
+    magic_link_code = db.Column(db.String(55), nullable=False, unique=True)
+
+    valid_from = db.Column(db.DateTime, nullable=True)
+    valid_to = db.Column(db.DateTime, nullable=True)
+
+    specialist_args = db.Column(JSONB, nullable=True)
+
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+    def __repr__(self):
+        return f"<SpecialistMagicLink {self.specialist_id} {self.magic_link_code}>"

common/models/monitoring.py

@@ -1,21 +0,0 @@
-from common.extensions import db
-
-
-class BusinessEventLog(db.Model):
-    __bind_key__ = 'public'
-    __table_args__ = {'schema': 'public'}
-
-    id = db.Column(db.Integer, primary_key=True)
-    timestamp = db.Column(db.DateTime, nullable=False)
-    event_type = db.Column(db.String(50), nullable=False)
-    tenant_id = db.Column(db.Integer, nullable=False)
-    trace_id = db.Column(db.String(50), nullable=False)
-    span_id = db.Column(db.String(50))
-    span_name = db.Column(db.String(50))
-    parent_span_id = db.Column(db.String(50))
-    document_version_id = db.Column(db.Integer)
-    chat_session_id = db.Column(db.String(50))
-    interaction_id = db.Column(db.Integer)
-    environment = db.Column(db.String(20))
-    message = db.Column(db.Text)
-    # Add any other fields relevant for invoicing or warnings

common/models/user.py

@@ -1,8 +1,12 @@
+from datetime import date
+
 from common.extensions import db
 from flask_security import UserMixin, RoleMixin
-from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 import sqlalchemy as sa
 
+from common.models.entitlements import License
+
 
 class Tenant(db.Model):
     """Tenant model"""
@@ -16,53 +20,34 @@ class Tenant(db.Model):
 
     # company Information
     id = db.Column(db.Integer, primary_key=True)
+    code = db.Column(db.String(50), unique=True, nullable=True)
     name = db.Column(db.String(80), unique=True, nullable=False)
     website = db.Column(db.String(255), nullable=True)
     timezone = db.Column(db.String(50), nullable=True, default='UTC')
-    rag_context = db.Column(db.Text, nullable=True)
     type = db.Column(db.String(20), nullable=True, server_default='Active')
 
     # language information
     default_language = db.Column(db.String(2), nullable=True)
     allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True)
 
-    # LLM specific choices
-    embedding_model = db.Column(db.String(50), nullable=True)
-    llm_model = db.Column(db.String(50), nullable=True)
-
-    # Embedding variables
-    html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
-    html_end_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'li'])
-    html_included_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
-    html_excluded_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
-    html_excluded_classes = db.Column(ARRAY(sa.String(200)), nullable=True)
-
-    min_chunk_size = db.Column(db.Integer, nullable=True, default=2000)
-    max_chunk_size = db.Column(db.Integer, nullable=True, default=3000)
-
-    # Embedding search variables
-    es_k = db.Column(db.Integer, nullable=True, default=5)
-    es_similarity_threshold = db.Column(db.Float, nullable=True, default=0.7)
-
-    # Chat variables
-    chat_RAG_temperature = db.Column(db.Float, nullable=True, default=0.3)
-    chat_no_RAG_temperature = db.Column(db.Float, nullable=True, default=0.5)
-    fallback_algorithms = db.Column(ARRAY(sa.String(50)), nullable=True)
-
-    # Licensing Information
-    license_start_date = db.Column(db.Date, nullable=True)
-    license_end_date = db.Column(db.Date, nullable=True)
-    allowed_monthly_interactions = db.Column(db.Integer, nullable=True)
-    encrypted_chat_api_key = db.Column(db.String(500), nullable=True)
-    encrypted_api_key = db.Column(db.String(500), nullable=True)
-
-    # Tuning enablers
-    embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
-    rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
+   # Entitlements
+    currency = db.Column(db.String(20), nullable=True)
+    storage_dirty = db.Column(db.Boolean, nullable=True, default=False)
 
     # Relations
     users = db.relationship('User', backref='tenant')
     domains = db.relationship('TenantDomain', backref='tenant')
+    licenses = db.relationship('License', back_populates='tenant')
+    license_usages = db.relationship('LicenseUsage', backref='tenant')
+
+    @property
+    def current_license(self):
+        today = date.today()
+        return License.query.filter(
+            License.tenant_id == self.id,
+            License.start_date <= today,
+            (License.end_date.is_(None) | (License.end_date >= today))
+        ).order_by(License.start_date.desc()).first()
 
     def __repr__(self):
         return f"<Tenant {self.id}: {self.name}>"
@@ -73,29 +58,10 @@ class Tenant(db.Model):
             'name': self.name,
             'website': self.website,
             'timezone': self.timezone,
-            'rag_context': self.rag_context,
             'type': self.type,
             'default_language': self.default_language,
             'allowed_languages': self.allowed_languages,
-            'embedding_model': self.embedding_model,
-            'llm_model': self.llm_model,
-            'html_tags': self.html_tags,
-            'html_end_tags': self.html_end_tags,
-            'html_included_elements': self.html_included_elements,
-            'html_excluded_elements': self.html_excluded_elements,
-            'html_excluded_classes': self.html_excluded_classes,
-            'min_chunk_size': self.min_chunk_size,
-            'max_chunk_size': self.max_chunk_size,
-            'es_k': self.es_k,
-            'es_similarity_threshold': self.es_similarity_threshold,
-            'chat_RAG_temperature': self.chat_RAG_temperature,
-            'chat_no_RAG_temperature': self.chat_no_RAG_temperature,
-            'fallback_algorithms': self.fallback_algorithms,
-            'license_start_date': self.license_start_date,
-            'license_end_date': self.license_end_date,
-            'allowed_monthly_interactions': self.allowed_monthly_interactions,
-            'embed_tuning': self.embed_tuning,
-            'rag_tuning': self.rag_tuning,
+            'currency': self.currency,
         }
 
 
@@ -128,6 +94,7 @@ class User(db.Model, UserMixin):
 
     # User Information
     id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
     user_name = db.Column(db.String(80), unique=True, nullable=False)
     email = db.Column(db.String(255), unique=True, nullable=False)
     password = db.Column(db.String(255), nullable=True)
@@ -137,6 +104,8 @@ class User(db.Model, UserMixin):
     fs_uniquifier = db.Column(db.String(255), unique=True, nullable=False)
     confirmed_at = db.Column(db.DateTime, nullable=True)
     valid_to = db.Column(db.Date, nullable=True)
+    is_primary_contact = db.Column(db.Boolean, nullable=True, default=False)
+    is_financial_contact = db.Column(db.Boolean, nullable=True, default=False)
 
     # Security Trackable Information
     last_login_at = db.Column(db.DateTime, nullable=True)
@@ -147,7 +116,6 @@ class User(db.Model, UserMixin):
 
     # Relations
     roles = db.relationship('Role', secondary=RolesUsers.__table__, backref=db.backref('users', lazy='dynamic'))
-    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
 
     def __repr__(self):
         return '<User %r>' % self.user_name
@@ -170,10 +138,166 @@ class TenantDomain(db.Model):
 
     # Versioning Information
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
-    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=False)
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=False)
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
-    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
 
     def __repr__(self):
         return f"<TenantDomain {self.id}: {self.domain}>"
 
+
+class TenantProject(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    services = db.Column(ARRAY(sa.String(50)), nullable=False)
+    encrypted_api_key = db.Column(db.String(500), nullable=True)
+    visual_api_key = db.Column(db.String(20), nullable=True)
+    active = db.Column(db.Boolean, nullable=False, default=True)
+    responsible_email = db.Column(db.String(255), nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+    # Relations
+    tenant = db.relationship('Tenant', backref='projects')
+
+    def __repr__(self):
+        return f"<TenantProject {self.id}: {self.name}>"
+
+
+class TenantMake(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    active = db.Column(db.Boolean, nullable=False, default=True)
+    website = db.Column(db.String(255), nullable=True)
+    logo_url = db.Column(db.String(255), nullable=True)
+
+    # Chat customisation options
+    chat_customisation_options = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'))
+
+
+class Partner(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False, unique=True)
+    code = db.Column(db.String(50), unique=True, nullable=False)
+
+    # Basic information
+    logo_url = db.Column(db.String(255), nullable=True)
+    active = db.Column(db.Boolean, default=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    # Relationships
+    services = db.relationship('PartnerService', back_populates='partner')
+    tenant = db.relationship('Tenant', backref=db.backref('partner', uselist=False))
+
+    def to_dict(self):
+        services_info = []
+        for service in self.services:
+            services_info.append({
+                'id': service.id,
+                'name': service.name,
+                'description': service.description,
+                'type': service.type,
+                'type_version': service.type_version,
+                'active': service.active,
+                'configuration': service.configuration,
+                'permissions': service.permissions,
+            })
+        return {
+            'id': self.id,
+            'tenant_id': self.tenant_id,
+            'code': self.code,
+            'logo_url': self.logo_url,
+            'active': self.active,
+            'name': self.tenant.name,
+            'services': services_info
+        }
+
+
+class PartnerService(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    partner_id = db.Column(db.Integer, db.ForeignKey('public.partner.id'), nullable=False)
+
+    # Basic info
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+
+    # Service type with versioning (similar to your specialist/retriever pattern)
+    type = db.Column(db.String(50), nullable=False)  # REFERRAL, KNOWLEDGE, SPECIALIST, IMPLEMENTATION, WHITE_LABEL
+    type_version = db.Column(db.String(20), nullable=False, default="1.0.0")
+
+    # Status
+    active = db.Column(db.Boolean, default=True)
+
+    # Dynamic configuration specific to this service - using JSONB like your other models
+    configuration = db.Column(db.JSON, nullable=True)
+    permissions = db.Column(db.JSON, nullable=True)
+
+    # For services that need to track shared resources
+    system_metadata = db.Column(db.JSON, nullable=True)
+    user_metadata = db.Column(db.JSON, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    # Relationships
+    partner = db.relationship('Partner', back_populates='services')
+    license_tiers = db.relationship('PartnerServiceLicenseTier', back_populates='partner_service')
+
+
+class PartnerTenant(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    partner_service_id = db.Column(db.Integer, db.ForeignKey('public.partner_service.id'), primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), primary_key=True)
+
+    # JSONB for flexible configuration specific to this relationship
+    configuration = db.Column(db.JSON, nullable=True)
+
+    # Tracking
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+
+class SpecialistMagicLinkTenant(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    magic_link_code = db.Column(db.String(55), primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)

common/services/entitlements/__init__.py

@@ -0,0 +1,9 @@
+from common.services.entitlements.license_period_services import LicensePeriodServices
+from common.services.entitlements.license_usage_services import LicenseUsageServices
+from common.services.entitlements.license_tier_services import LicenseTierServices
+
+__all__ = [
+    'LicensePeriodServices',
+    'LicenseUsageServices',
+    'LicenseTierServices'
+]

common/services/entitlements/license_period_services.py

@@ -0,0 +1,247 @@
+from dateutil.relativedelta import relativedelta
+from datetime import datetime as dt, timezone as tz, timedelta
+from flask import current_app
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.expression import and_
+
+from common.extensions import db
+from common.models.entitlements import LicensePeriod, License, PeriodStatus, LicenseUsage
+from common.utils.eveai_exceptions import EveAILicensePeriodsExceeded, EveAIPendingLicensePeriod, EveAINoActiveLicense
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+
+
+class LicensePeriodServices:
+    @staticmethod
+    def find_current_license_period_for_usage(tenant_id: int) -> LicensePeriod:
+        """
+        Find the current license period for a tenant. It ensures the status of the different license periods are adapted
+        when required, and a LicenseUsage object is created if required.
+
+        Args:
+            tenant_id: The ID of the tenant to find the license period for
+
+        Raises:
+            EveAIException: and derived classes
+        """
+        try:
+            current_app.logger.debug(f"Finding current license period for tenant {tenant_id}")
+            current_date = dt.now(tz.utc).date()
+            license_period = (db.session.query(LicensePeriod)
+                              .filter_by(tenant_id=tenant_id)
+                              .filter(and_(LicensePeriod.period_start <= current_date,
+                                           LicensePeriod.period_end >= current_date))
+                              .first())
+            current_app.logger.debug(f"End searching for license period for tenant {tenant_id} ")
+            if not license_period:
+                current_app.logger.debug(f"No license period found for tenant {tenant_id} on date {current_date}")
+                license_period = LicensePeriodServices._create_next_license_period_for_usage(tenant_id)
+                current_app.logger.debug(f"Created license period {license_period.id} for tenant {tenant_id}")
+            if license_period:
+                current_app.logger.debug(f"Found license period {license_period.id} for tenant {tenant_id} "
+                                         f"with status {license_period.status}")
+                match license_period.status:
+                    case PeriodStatus.UPCOMING:
+                        current_app.logger.debug(f"In upcoming state")
+                        LicensePeriodServices._complete_last_license_period(tenant_id=tenant_id)
+                        current_app.logger.debug(f"Completed last license period for tenant {tenant_id}")
+                        LicensePeriodServices._activate_license_period(license_period=license_period)
+                        current_app.logger.debug(f"Activated license period {license_period.id} for tenant {tenant_id}")
+                        if not license_period.license_usage:
+                            new_license_usage = LicenseUsage(
+                                tenant_id=tenant_id,
+                            )
+                            new_license_usage.license_period = license_period
+                            try:
+                                db.session.add(new_license_usage)
+                                db.session.commit()
+
+                            except SQLAlchemyError as e:
+                                db.session.rollback()
+                                current_app.logger.error(
+                                    f"Error creating new license usage for license period "
+                                    f"{license_period.id}: {str(e)}")
+                                raise e
+                        if license_period.status == PeriodStatus.ACTIVE:
+                            return license_period
+                        else:
+                            # Status is PENDING, so no prepaid payment received. There is no license period we can use.
+                            # We allow for a delay of 5 days before raising an exception.
+                            current_date = dt.now(tz.utc).date()
+                            delta = abs(current_date - license_period.period_start)
+                            if delta > timedelta(days=current_app.config.get('ENTITLEMENTS_MAX_PENDING_DAYS', 5)):
+                                raise EveAIPendingLicensePeriod()
+                    case PeriodStatus.ACTIVE:
+                        return license_period
+                    case PeriodStatus.PENDING:
+                        return license_period
+            else:
+                raise EveAILicensePeriodsExceeded(license_id=None)
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error finding current license period for tenant {tenant_id}: {str(e)}")
+            raise e
+        except Exception as e:
+            raise e
+
+    @staticmethod
+    def _create_next_license_period_for_usage(tenant_id) -> LicensePeriod:
+        """
+        Create a new period for this license using the current license configuration
+
+        Args:
+            tenant_id: The ID of the tenant to create the period for
+
+        Returns:
+            LicensePeriod: The newly created license period
+        """
+        current_date = dt.now(tz.utc).date()
+
+        # Zoek de actieve licentie voor deze tenant op de huidige datum
+        the_license = (db.session.query(License)
+                      .filter_by(tenant_id=tenant_id)
+                      .filter(License.start_date <= current_date)
+                      .filter(License.end_date >= current_date)
+                      .first())
+
+        if not the_license:
+            current_app.logger.error(f"No active license found for tenant {tenant_id} on date {current_date}")
+            raise EveAINoActiveLicense(tenant_id=tenant_id)
+        else:
+            current_app.logger.debug(f"Found active license {the_license.id} for tenant {tenant_id} "
+                                    f"on date {current_date}")
+
+        next_period_number = 1
+        if the_license.periods:
+            # If there are existing periods, get the next sequential number
+            next_period_number = max(p.period_number for p in the_license.periods) + 1
+        current_app.logger.debug(f"Next period number for tenant {tenant_id} is {next_period_number}")
+
+        if next_period_number > the_license.nr_of_periods:
+            raise EveAILicensePeriodsExceeded(license_id=the_license.id)
+
+        new_license_period = LicensePeriod(
+            license_id=the_license.id,
+            tenant_id=tenant_id,
+            period_number=next_period_number,
+            period_start=the_license.start_date + relativedelta(months=next_period_number-1),
+            period_end=the_license.end_date + relativedelta(months=next_period_number, days=-1),
+            status=PeriodStatus.UPCOMING,
+            upcoming_at=dt.now(tz.utc),
+        )
+        set_logging_information(new_license_period, dt.now(tz.utc))
+
+        try:
+            current_app.logger.debug(f"Creating next license period for tenant {tenant_id} ")
+            db.session.add(new_license_period)
+            db.session.commit()
+            current_app.logger.info(f"Created next license period for tenant {tenant_id} "
+                                    f"with id {new_license_period.id}")
+            return new_license_period
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error creating next license period for tenant {tenant_id}: {str(e)}")
+            raise e
+
+    @staticmethod
+    def _activate_license_period(license_period_id: int = None, license_period: LicensePeriod = None) -> LicensePeriod:
+        """
+        Activate a license period
+
+        Args:
+            license_period_id: The ID of the license period to activate (optional if license_period is provided)
+            license_period: The LicensePeriod object to activate (optional if license_period_id is provided)
+
+        Returns:
+            LicensePeriod: The activated license period object
+
+        Raises:
+            ValueError: If neither license_period_id nor license_period is provided
+        """
+        current_app.logger.debug(f"Activating license period")
+        if license_period is None and license_period_id is None:
+            raise ValueError("Either license_period_id or license_period must be provided")
+
+        # Get a license period object if only ID was provided
+        if license_period is None:
+            current_app.logger.debug(f"Getting license period {license_period_id} to activate")
+            license_period = LicensePeriod.query.get_or_404(license_period_id)
+
+        if license_period.pending_at is not None:
+            license_period.pending_at = dt.now(tz.utc)
+        license_period.status = PeriodStatus.PENDING
+        if license_period.prepaid_payment:
+            # There is a payment received for the given period
+            license_period.active_at = dt.now(tz.utc)
+            license_period.status = PeriodStatus.ACTIVE
+
+        # Copy snapshot fields from the license to the period
+        the_license = License.query.get_or_404(license_period.license_id)
+        license_period.currency = the_license.currency
+        license_period.basic_fee = the_license.basic_fee
+        license_period.max_storage_mb = the_license.max_storage_mb
+        license_period.additional_storage_price = the_license.additional_storage_price
+        license_period.additional_storage_bucket = the_license.additional_storage_bucket
+        license_period.included_embedding_mb = the_license.included_embedding_mb
+        license_period.additional_embedding_price = the_license.additional_embedding_price
+        license_period.additional_embedding_bucket = the_license.additional_embedding_bucket
+        license_period.included_interaction_tokens = the_license.included_interaction_tokens
+        license_period.additional_interaction_token_price = the_license.additional_interaction_token_price
+        license_period.additional_interaction_bucket = the_license.additional_interaction_bucket
+        license_period.additional_storage_allowed = the_license.additional_storage_allowed
+        license_period.additional_embedding_allowed = the_license.additional_embedding_allowed
+        license_period.additional_interaction_allowed = the_license.additional_interaction_allowed
+
+        update_logging_information(license_period, dt.now(tz.utc))
+
+        if not license_period.license_usage:
+            license_period.license_usage = LicenseUsage(
+                tenant_id=license_period.tenant_id,
+                license_period_id=license_period.id,
+            )
+
+        license_period.license_usage.recalculate_storage()
+
+        try:
+            db.session.add(license_period)
+            db.session.add(license_period.license_usage)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error activating license period {license_period_id}: {str(e)}")
+            raise e
+
+        return license_period
+
+    @staticmethod
+    def _complete_last_license_period(tenant_id) -> None:
+        """
+        Complete the active or pending license period for a tenant. This is done by setting the status to COMPLETED.
+
+        Args:
+            tenant_id: De ID van de tenant
+        """
+        # Zoek de licenseperiode voor deze tenant met status ACTIVE of PENDING
+        active_period = (
+            db.session.query(LicensePeriod)
+            .filter_by(tenant_id=tenant_id)
+            .filter(LicensePeriod.status.in_([PeriodStatus.ACTIVE, PeriodStatus.PENDING]))
+            .first()
+        )
+
+        # Als er geen actieve periode gevonden is, hoeven we niets te doen
+        if not active_period:
+            return
+
+        # Zet de gevonden periode op COMPLETED
+        active_period.status = PeriodStatus.COMPLETED
+        active_period.completed_at = dt.now(tz.utc)
+        update_logging_information(active_period, dt.now(tz.utc))
+
+        try:
+            db.session.add(active_period)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error completing period {active_period.id} for {tenant_id}: {str(e)}")
+            raise e

common/services/entitlements/license_tier_services.py

@@ -0,0 +1,67 @@
+from flask import session, flash, current_app
+from datetime import datetime as dt, timezone as tz
+
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db
+from common.models.entitlements import PartnerServiceLicenseTier
+from common.models.user import Partner
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService, EveAINoSessionPartner
+from common.utils.model_logging_utils import set_logging_information
+
+
+class LicenseTierServices:
+    @staticmethod
+    def associate_license_tier_with_partner(license_tier_id):
+        """Associate a license tier with a partner"""
+        try:
+            partner_id = session['partner']['id']
+            # Get partner service (MANAGEMENT_SERVICE type)
+            partner = Partner.query.get(partner_id)
+            if not partner:
+                raise EveAINoSessionPartner()
+
+            # Find a management service for this partner
+            management_service = next((service for service in session['partner']['services']
+                                       if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+
+            if not management_service:
+                flash("Cannot associate license tier with partner. No management service defined for partner", "danger")
+                current_app.logger.error(f"No Management Service defined for partner {partner_id}"
+                                         f"trying to associate license tier {license_tier_id}.")
+                raise EveAINoManagementPartnerService()
+            # Check if the association already exists
+            existing_association = PartnerServiceLicenseTier.query.filter_by(
+                partner_service_id=management_service['id'],
+                license_tier_id=license_tier_id
+            ).first()
+
+            if existing_association:
+                # Association already exists, nothing to do
+                flash("License tier was already associated with partner", "info")
+                current_app.logger.info(f"Association between partner service {management_service['id']} and "
+                                        f"license tier {license_tier_id} already exists.")
+                return
+
+            # Create the association
+            association = PartnerServiceLicenseTier(
+                partner_service_id=management_service['id'],
+                license_tier_id=license_tier_id
+            )
+            set_logging_information(association, dt.now(tz.utc))
+
+            db.session.add(association)
+            db.session.commit()
+
+            flash("Successfully associated license tier to partner", "success")
+            current_app.logger.info(f"Successfully associated license tier {license_tier_id} with "
+                                    f"partner service {management_service['id']}")
+
+            return True
+
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash("Failed to associated license tier with partner service due to an internal error. "
+                  "Please contact the System Administrator", "danger")
+            current_app.logger.error(f"Error associating license tier {license_tier_id} with partner: {str(e)}")
+            raise e

common/services/entitlements/license_usage_services.py

@@ -0,0 +1,143 @@
+from dateutil.relativedelta import relativedelta
+from flask import session, current_app, flash
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.expression import text
+
+from common.extensions import db, cache_manager
+from common.models.entitlements import PartnerServiceLicenseTier, License, LicenseUsage, LicensePeriod, PeriodStatus
+from common.models.user import Partner, PartnerTenant
+from common.services.entitlements import LicensePeriodServices
+from common.utils.database import Database
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService, EveAINoActiveLicense, \
+    EveAIStorageQuotaExceeded, EveAIEmbeddingQuotaExceeded, EveAIInteractionQuotaExceeded, EveAILicensePeriodsExceeded, \
+    EveAIException
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+from datetime import datetime as dt, timezone as tz
+
+from common.utils.security_utils import current_user_has_role
+
+
+class LicenseUsageServices:
+    @staticmethod
+    def check_storage_and_embedding_quota(tenant_id: int, file_size_mb: float) -> None:
+        """
+        Check if a tenant can add a new document without exceeding storage and embedding quotas
+
+        Args:
+            tenant_id: ID of the tenant
+            file_size_mb: Size of the file in MB
+
+        Raises:
+            EveAIStorageQuotaExceeded: If storage quota would be exceeded
+            EveAIEmbeddingQuotaExceeded: If embedding quota would be exceeded
+            EveAINoActiveLicense: If no active license is found
+            EveAIException: For other errors
+        """
+        # Get active license period
+        license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
+        # Early return if both overruns are allowed - no need to check usage at all
+        if license_period.additional_storage_allowed and license_period.additional_embedding_allowed:
+            return
+
+        # Check storage quota only if overruns are not allowed
+        if not license_period.additional_storage_allowed:
+            LicenseUsageServices._validate_storage_quota(license_period, file_size_mb)
+
+        # Check embedding quota only if overruns are not allowed
+        if not license_period.additional_embedding_allowed:
+            LicenseUsageServices._validate_embedding_quota(license_period, file_size_mb)
+
+    @staticmethod
+    def check_embedding_quota(tenant_id: int, file_size_mb: float) -> None:
+        """
+        Check if a tenant can re-embed a document without exceeding embedding quota
+
+        Args:
+            tenant_id: ID of the tenant
+            file_size_mb: Size of the file in MB
+
+        Raises:
+            EveAIEmbeddingQuotaExceeded: If embedding quota would be exceeded
+            EveAINoActiveLicense: If no active license is found
+            EveAIException: For other errors
+        """
+        # Get active license period
+        license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
+        # Early return if both overruns are allowed - no need to check usage at all
+        if license_period.additional_embedding_allowed:
+            return
+
+        # Check embedding quota
+        LicenseUsageServices._validate_embedding_quota(license_period, file_size_mb)
+
+    @staticmethod
+    def check_interaction_quota(tenant_id: int) -> None:
+        """
+        Check if a tenant can execute a specialist without exceeding interaction quota. As it is impossible to estimate
+        the number of interaction tokens, we only check if the interaction quota are exceeded. So we might have a
+        limited overrun.
+
+        Args:
+            tenant_id: ID of the tenant
+
+        Raises:
+            EveAIInteractionQuotaExceeded: If interaction quota would be exceeded
+            EveAINoActiveLicense: If no active license is found
+            EveAIException: For other errors
+        """
+        # Get active license period
+        license_period = LicensePeriodServices.find_current_license_period_for_usage(tenant_id)
+        # Early return if both overruns are allowed - no need to check usage at all
+        if license_period.additional_interaction_allowed:
+            return
+
+        # Convert tokens to M tokens and check interaction quota
+        LicenseUsageServices._validate_interaction_quota(license_period)
+
+    @staticmethod
+    def _validate_storage_quota(license_period: LicensePeriod, additional_mb: float) -> None:
+        """Check storage quota and raise exception if exceeded"""
+        current_storage = license_period.license_usage.storage_mb_used or 0
+        projected_storage = current_storage + additional_mb
+        max_storage = license_period.max_storage_mb
+
+        # Hard limit check (we only get here if overruns are NOT allowed)
+        if projected_storage > max_storage:
+            raise EveAIStorageQuotaExceeded(
+                current_usage=current_storage,
+                limit=max_storage,
+                additional=additional_mb
+            )
+
+    @staticmethod
+    def _validate_embedding_quota(license_period: LicensePeriod, additional_mb: float) -> None:
+        """Check embedding quota and raise exception if exceeded"""
+        current_embedding = license_period.license_usage.embedding_mb_used or 0
+        projected_embedding = current_embedding + additional_mb
+        max_embedding = license_period.included_embedding_mb
+
+        # Hard limit check (we only get here if overruns are NOT allowed)
+        if projected_embedding > max_embedding:
+            raise EveAIEmbeddingQuotaExceeded(
+                current_usage=current_embedding,
+                limit=max_embedding,
+                additional=additional_mb
+            )
+
+    @staticmethod
+    def _validate_interaction_quota(license_period) -> None:
+        """Check interaction quota and raise exception if exceeded (tokens in millions). We might have an overrun!"""
+        current_tokens = license_period.license_usage.interaction_total_tokens_used / 1_000_000 or 0
+        max_tokens = license_period.included_interaction_tokens
+
+        # Hard limit check (we only get here if overruns are NOT allowed)
+        if current_tokens > max_tokens:
+            raise EveAIInteractionQuotaExceeded(
+                current_usage=current_tokens,
+                limit=max_tokens
+            )
+
+
+
+
+

common/services/interaction/specialist_services.py

@@ -0,0 +1,222 @@
+import uuid
+from datetime import datetime as dt, timezone as tz
+from typing import Dict, Any, Tuple, Optional
+from flask import current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db, cache_manager
+from common.models.interaction import (
+    Specialist, EveAIAgent, EveAITask, EveAITool
+)
+from common.utils.celery_utils import current_celery
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+
+
+class SpecialistServices:
+    @staticmethod
+    def start_session() -> str:
+        return f"CHAT_SESSION_{uuid.uuid4()}"
+
+    @staticmethod
+    def execute_specialist(tenant_id, specialist_id, specialist_arguments, session_id, user_timezone) -> Dict[str, Any]:
+        task = current_celery.send_task(
+            'execute_specialist',
+            args=[tenant_id,
+                  specialist_id,
+                  specialist_arguments,
+                  session_id,
+                  user_timezone,
+                  ],
+            queue='llm_interactions'
+        )
+
+        return {
+            'task_id': task.id,
+            'status': 'queued',
+        }
+
+    @staticmethod
+    def initialize_specialist(specialist_id: int, specialist_type: str, specialist_version: str):
+        """
+        Initialize an agentic specialist by creating all its components based on configuration.
+
+        Args:
+            specialist_id: ID of the specialist to initialize
+            specialist_type: Type of the specialist
+            specialist_version: Version of the specialist type to use
+
+        Raises:
+            ValueError: If specialist not found or invalid configuration
+            SQLAlchemyError: If database operations fail
+        """
+        config = cache_manager.specialists_config_cache.get_config(specialist_type, specialist_version)
+        if not config:
+            raise ValueError(f"No configuration found for {specialist_type} version {specialist_version}")
+        if config['framework'] == 'langchain':
+            pass    # Langchain does not require additional items to be initialized. All configuration is in the specialist.
+
+        specialist = Specialist.query.get(specialist_id)
+        if not specialist:
+            raise ValueError(f"Specialist with ID {specialist_id} not found")
+
+        if config['framework'] == 'crewai':
+            SpecialistServices.initialize_crewai_specialist(specialist, config)
+
+    @staticmethod
+    def initialize_crewai_specialist(specialist: Specialist, config: Dict[str, Any]):
+        timestamp = dt.now(tz=tz.utc)
+
+        try:
+            # Initialize agents
+            if 'agents' in config:
+                for agent_config in config['agents']:
+                    SpecialistServices._create_agent(
+                        specialist_id=specialist.id,
+                        agent_type=agent_config['type'],
+                        agent_version=agent_config['version'],
+                        name=agent_config.get('name'),
+                        description=agent_config.get('description'),
+                        timestamp=timestamp
+                    )
+
+            # Initialize tasks
+            if 'tasks' in config:
+                for task_config in config['tasks']:
+                    SpecialistServices._create_task(
+                        specialist_id=specialist.id,
+                        task_type=task_config['type'],
+                        task_version=task_config['version'],
+                        name=task_config.get('name'),
+                        description=task_config.get('description'),
+                        timestamp=timestamp
+                    )
+
+            # Initialize tools
+            if 'tools' in config:
+                for tool_config in config['tools']:
+                    SpecialistServices._create_tool(
+                        specialist_id=specialist.id,
+                        tool_type=tool_config['type'],
+                        tool_version=tool_config['version'],
+                        name=tool_config.get('name'),
+                        description=tool_config.get('description'),
+                        timestamp=timestamp
+                    )
+
+            db.session.commit()
+            current_app.logger.info(f"Successfully initialized crewai specialist {specialist.id}")
+
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Database error initializing crewai specialist {specialist.id}: {str(e)}")
+            raise
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error initializing crewai specialist {specialist.id}: {str(e)}")
+            raise
+
+    @staticmethod
+    def _create_agent(
+            specialist_id: int,
+            agent_type: str,
+            agent_version: str,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            timestamp: Optional[dt] = None
+    ) -> EveAIAgent:
+        """Create an agent with the given configuration."""
+        if timestamp is None:
+            timestamp = dt.now(tz=tz.utc)
+
+        # Get agent configuration from cache
+        agent_config = cache_manager.agents_config_cache.get_config(agent_type, agent_version)
+
+        agent = EveAIAgent(
+            specialist_id=specialist_id,
+            name=name or agent_config.get('name', agent_type),
+            description=description or agent_config.get('metadata').get('description', ''),
+            type=agent_type,
+            type_version=agent_version,
+            role=None,
+            goal=None,
+            backstory=None,
+            tuning=False,
+            configuration=None,
+            arguments=None
+        )
+
+        set_logging_information(agent, timestamp)
+
+        db.session.add(agent)
+        current_app.logger.info(f"Created agent {agent.id} of type {agent_type}")
+        return agent
+
+    @staticmethod
+    def _create_task(
+            specialist_id: int,
+            task_type: str,
+            task_version: str,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            timestamp: Optional[dt] = None
+    ) -> EveAITask:
+        """Create a task with the given configuration."""
+        if timestamp is None:
+            timestamp = dt.now(tz=tz.utc)
+
+        # Get task configuration from cache
+        task_config = cache_manager.tasks_config_cache.get_config(task_type, task_version)
+
+        task = EveAITask(
+            specialist_id=specialist_id,
+            name=name or task_config.get('name', task_type),
+            description=description or task_config.get('metadata').get('description', ''),
+            type=task_type,
+            type_version=task_version,
+            task_description=None,
+            expected_output=None,
+            tuning=False,
+            configuration=None,
+            arguments=None,
+            context=None,
+            asynchronous=False,
+        )
+
+        set_logging_information(task, timestamp)
+
+        db.session.add(task)
+        current_app.logger.info(f"Created task {task.id} of type {task_type}")
+        return task
+
+    @staticmethod
+    def _create_tool(
+            specialist_id: int,
+            tool_type: str,
+            tool_version: str,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            timestamp: Optional[dt] = None
+    ) -> EveAITool:
+        """Create a tool with the given configuration."""
+        if timestamp is None:
+            timestamp = dt.now(tz=tz.utc)
+
+        # Get tool configuration from cache
+        tool_config = cache_manager.tools_config_cache.get_config(tool_type, tool_version)
+
+        tool = EveAITool(
+            specialist_id=specialist_id,
+            name=name or tool_config.get('name', tool_type),
+            description=description or tool_config.get('metadata').get('description', ''),
+            type=tool_type,
+            type_version=tool_version,
+            tuning=False,
+            configuration=None,
+            arguments=None,
+        )
+
+        set_logging_information(tool, timestamp)
+
+        db.session.add(tool)
+        current_app.logger.info(f"Created tool {tool.id} of type {tool_type}")
+        return tool

common/services/user/__init__.py

@@ -0,0 +1,5 @@
+from common.services.user.user_services import UserServices
+from common.services.user.partner_services import PartnerServices
+from common.services.user.tenant_services import TenantServices
+
+__all__ = ['UserServices', 'PartnerServices', 'TenantServices']

common/services/user/partner_services.py

@@ -0,0 +1,47 @@
+from typing import List
+
+from flask import session
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.models.entitlements import PartnerServiceLicenseTier
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService, EveAINoSessionPartner
+
+from common.utils.security_utils import current_user_has_role
+
+
+class PartnerServices:
+    @staticmethod
+    def get_allowed_license_tier_ids() -> List[int]:
+        """
+        Retrieve IDs of all License Tiers associated with the partner's management service
+
+        Returns:
+            List of license tier IDs
+
+        Raises:
+            EveAINoSessionPartner: If no partner is in the session
+            EveAINoManagementPartnerService: If partner has no management service
+        """
+        partner = session.get("partner", None)
+        if not partner:
+            raise EveAINoSessionPartner()
+
+        # Find a management service for this partner
+        management_service = next((service for service in session['partner']['services']
+                                   if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+        if not management_service:
+            raise EveAINoManagementPartnerService()
+        management_service_id = management_service['id']
+
+        # Query for all license tiers associated with this management service
+        associations = PartnerServiceLicenseTier.query.filter_by(
+            partner_service_id=management_service_id
+        ).all()
+
+        # Extract the license tier IDs
+        license_tier_ids = [assoc.license_tier_id for assoc in associations]
+
+        return license_tier_ids
+
+
+

common/services/user/tenant_services.py

@@ -0,0 +1,175 @@
+from typing import Dict, List
+
+from flask import session, current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db, cache_manager
+from common.models.user import Partner, PartnerTenant, PartnerService, Tenant
+from common.utils.eveai_exceptions import EveAINoManagementPartnerService
+from common.utils.model_logging_utils import set_logging_information
+from datetime import datetime as dt, timezone as tz
+
+from common.utils.security_utils import current_user_has_role
+
+
+class TenantServices:
+    @staticmethod
+    def associate_tenant_with_partner(tenant_id):
+        """Associate a tenant with a partner"""
+        try:
+            partner_id = session['partner']['id']
+            # Get partner service (MANAGEMENT_SERVICE type)
+            partner = Partner.query.get(partner_id)
+            if not partner:
+                return
+
+            # Find a management service for this partner
+            management_service = next((service for service in session['partner']['services']
+                                       if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+
+            if not management_service:
+                current_app.logger.error(f"No Management Service defined for partner {partner_id} "
+                                         f"while associating tenant {tenant_id} with partner.")
+                raise EveAINoManagementPartnerService()
+
+            # Create the association
+            tenant_partner = PartnerTenant(
+                partner_service_id=management_service['id'],
+                tenant_id=tenant_id,
+            )
+            set_logging_information(tenant_partner, dt.now(tz.utc))
+
+            db.session.add(tenant_partner)
+            db.session.commit()
+
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error associating tenant {tenant_id} with partner: {str(e)}")
+            raise e
+
+        @staticmethod
+        def get_available_types_for_tenant(tenant_id: int, config_type: str) -> Dict[str, Dict[str, str]]:
+            """
+            Get available configuration types for a tenant based on partner relationships
+
+            Args:
+                tenant_id: The tenant ID
+                config_type: The configuration type ('specialists', 'agents', 'tasks', etc.)
+
+            Returns:
+                Dictionary of available types for the tenant
+            """
+            # Get the appropriate cache handler based on config_type
+            cache_handler = None
+            if config_type == 'specialists':
+                cache_handler = cache_manager.specialists_types_cache
+            elif config_type == 'agents':
+                cache_handler = cache_manager.agents_types_cache
+            elif config_type == 'tasks':
+                cache_handler = cache_manager.tasks_types_cache
+            elif config_type == 'tools':
+                cache_handler = cache_manager.tools_types_cache
+            else:
+                raise ValueError(f"Unsupported config type: {config_type}")
+
+            # Get all types with their metadata (including partner info)
+            all_types = cache_handler.get_types()
+
+            # Filter to include:
+            # 1. Types with no partner (global)
+            # 2. Types with partners that have a SPECIALIST_SERVICE relationship with this tenant
+            available_partners = TenantServices.get_tenant_partner_names(tenant_id)
+
+            available_types = {
+                type_id: info for type_id, info in all_types.items()
+                if info.get('partner') is None or info.get('partner') in available_partners
+            }
+
+            return available_types
+
+        @staticmethod
+        def get_tenant_partner_names(tenant_id: int) -> List[str]:
+            """
+            Get names of partners that have a SPECIALIST_SERVICE relationship with this tenant
+
+            Args:
+                tenant_id: The tenant ID
+
+            Returns:
+                List of partner names (tenant names)
+            """
+            # Find all PartnerTenant relationships for this tenant
+            partner_names = []
+            try:
+                # Get all partner services of type SPECIALIST_SERVICE
+                specialist_services = (
+                    PartnerService.query
+                    .filter_by(type='SPECIALIST_SERVICE')
+                    .all()
+                )
+
+                if not specialist_services:
+                    return []
+
+                # Find tenant relationships with these services
+                partner_tenants = (
+                    PartnerTenant.query
+                    .filter_by(tenant_id=tenant_id)
+                    .filter(PartnerTenant.partner_service_id.in_([svc.id for svc in specialist_services]))
+                    .all()
+                )
+
+                # Get the partner names (their tenant names)
+                for pt in partner_tenants:
+                    partner_service = (
+                        PartnerService.query
+                        .filter_by(id=pt.partner_service_id)
+                        .first()
+                    )
+
+                    if partner_service:
+                        partner = Partner.query.get(partner_service.partner_id)
+                        if partner:
+                            # Get the tenant associated with this partner
+                            partner_tenant = Tenant.query.get(partner.tenant_id)
+                            if partner_tenant:
+                                partner_names.append(partner_tenant.name)
+
+            except SQLAlchemyError as e:
+                current_app.logger.error(f"Database error retrieving partner names: {str(e)}")
+
+            return partner_names
+
+        @staticmethod
+        def can_use_specialist_type(tenant_id: int, specialist_type: str) -> bool:
+            """
+            Check if a tenant can use a specific specialist type
+
+            Args:
+                tenant_id: The tenant ID
+                specialist_type: The specialist type ID
+
+            Returns:
+                True if the tenant can use the specialist type, False otherwise
+            """
+            # Get the specialist type definition
+            try:
+                specialist_types = cache_manager.specialists_types_cache.get_types()
+                specialist_def = specialist_types.get(specialist_type)
+
+                if not specialist_def:
+                    return False
+
+                # If it's a global specialist, anyone can use it
+                if specialist_def.get('partner') is None:
+                    return True
+
+                # If it's a partner-specific specialist, check if tenant has access
+                partner_name = specialist_def.get('partner')
+                available_partners = TenantServices.get_tenant_partner_names(tenant_id)
+
+                return partner_name in available_partners
+
+            except Exception as e:
+                current_app.logger.error(f"Error checking specialist type access: {str(e)}")
+                return False

common/services/user/user_services.py

@@ -0,0 +1,95 @@
+from flask import session
+
+from common.models.user import Partner, Role, PartnerTenant
+
+from common.utils.eveai_exceptions import EveAIRoleAssignmentException
+from common.utils.security_utils import current_user_has_role
+
+
+class UserServices:
+    @staticmethod
+    def get_assignable_roles():
+        """Retrieves roles that can be assigned to a user depending on the current user logged in,
+        and the active tenant for the session"""
+        current_tenant_id = session.get('tenant').get('id', None)
+        effective_role_names = []
+        if current_tenant_id == 1:
+            if current_user_has_role("Super User"):
+                effective_role_names.append("Super User")
+        elif current_tenant_id:
+            if current_user_has_role("Tenant Admin"):
+                effective_role_names.append("Tenant Admin")
+            if current_user_has_role("Partner Admin") or current_user_has_role("Super User"):
+                effective_role_names.append("Tenant Admin")
+                if session.get('partner'):
+                    if session.get('partner').get('tenant_id') == current_tenant_id:
+                        effective_role_names.append("Partner Admin")
+        effective_role_names = list(set(effective_role_names))
+        effective_roles = [(role.id, role.name) for role in
+                           Role.query.filter(Role.name.in_(effective_role_names)).all()]
+        return effective_roles
+
+    @staticmethod
+    def validate_role_assignments(role_ids):
+        """Validate a set of role assignments, raising exception for first invalid role"""
+        assignable_roles = UserServices.get_assignable_roles()
+        assignable_role_ids = {role[0] for role in assignable_roles}
+        role_id_set = set(role_ids)
+        return role_id_set.issubset(assignable_role_ids)
+
+    @staticmethod
+    def can_user_edit_tenant(tenant_id) -> bool:
+        if current_user_has_role('Super User'):
+            return True
+        elif current_user_has_role('Partner Admin'):
+            partner = session.get('partner', None)
+            if partner and partner["tenant_id"] == tenant_id:
+                return True
+            partner_service = next((service for service in session['partner']['services']
+                                    if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+            if not partner_service:
+                return False
+            else:
+                partner_tenant = PartnerTenant.query.filter(
+                    PartnerTenant.tenant_id == tenant_id,
+                    PartnerTenant.partner_service_id == partner_service['id'],
+                ).first()
+                if partner_tenant:
+                    return True
+                else:
+                    return False
+        else:
+            return False
+
+    @staticmethod
+    def can_user_create_tenant() -> bool:
+        if current_user_has_role('Super User'):
+            return True
+        elif current_user_has_role('Partner Admin'):
+            partner_id = session['partner']['id']
+            partner_service = next((service for service in session['partner']['services']
+                                    if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+            if not partner_service:
+                return False
+            else:
+                partner_permissions = partner_service.get('permissions', None)
+                return partner_permissions.get('can_create_tenant', False)
+        else:
+            return False
+
+    @staticmethod
+    def can_user_assign_license() -> bool:
+        if current_user_has_role('Super User'):
+            return True
+        elif current_user_has_role('Partner Admin'):
+            partner_id = session['partner']['id']
+            partner_service = next((service for service in session['partner']['services']
+                                    if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+            if not partner_service:
+                return False
+            else:
+                partner_permissions = partner_service.get('permissions', None)
+                return partner_permissions.get('can_assign_license', False)
+        else:
+            return False
+

common/utils/asset_utils.py

@@ -0,0 +1,62 @@
+from datetime import datetime as dt, timezone as tz
+
+from flask import current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import cache_manager, minio_client, db
+from common.models.interaction import EveAIAsset, EveAIAssetVersion
+from common.utils.model_logging_utils import set_logging_information
+
+
+def create_asset_stack(api_input, tenant_id):
+    type_version = cache_manager.assets_version_tree_cache.get_latest_version(api_input['type'])
+    api_input['type_version'] = type_version
+    new_asset = create_asset(api_input, tenant_id)
+    new_asset_version = create_version_for_asset(new_asset, tenant_id)
+    db.session.add(new_asset)
+    db.session.add(new_asset_version)
+
+    try:
+        db.session.commit()
+    except SQLAlchemyError as e:
+        current_app.logger.error(f"Could not add asset for tenant {tenant_id}: {str(e)}")
+        db.session.rollback()
+        raise e
+
+    return new_asset, new_asset_version
+
+
+def create_asset(api_input, tenant_id):
+    new_asset = EveAIAsset()
+    new_asset.name = api_input['name']
+    new_asset.description = api_input['description']
+    new_asset.type = api_input['type']
+    new_asset.type_version = api_input['type_version']
+    if api_input['valid_from'] and api_input['valid_from'] != '':
+        new_asset.valid_from = api_input['valid_from']
+    else:
+        new_asset.valid_from = dt.now(tz.utc)
+    new_asset.valid_to = api_input['valid_to']
+    set_logging_information(new_asset, dt.now(tz.utc))
+
+    return new_asset
+
+
+def create_version_for_asset(asset, tenant_id):
+    new_asset_version = EveAIAssetVersion()
+    new_asset_version.asset = asset
+    new_asset_version.bucket_name = minio_client.create_tenant_bucket(tenant_id)
+    set_logging_information(new_asset_version, dt.now(tz.utc))
+
+    return new_asset_version
+
+
+def add_asset_version_file(asset_version, field_name, file, tenant_id):
+    object_name, file_size = minio_client.upload_file(asset_version.bucket_name, asset_version.id, field_name,
+                                                      file.content_type)
+    # mark_tenant_storage_dirty(tenant_id)
+    # TODO - zorg ervoor dat de herberekening van storage onmiddellijk gebeurt!
+    return object_name
+
+
+

common/utils/business_event.py

@@ -1,15 +1,83 @@
 import os
+import time
 import uuid
-from contextlib import contextmanager
+from contextlib import contextmanager, asynccontextmanager
 from datetime import datetime
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 from datetime import datetime as dt, timezone as tz
-from portkey_ai import Portkey, Config
 import logging
 
+from flask import current_app
+from prometheus_client import Counter, Histogram, Gauge, Summary, push_to_gateway, REGISTRY
+
 from .business_event_context import BusinessEventContext
-from common.models.monitoring import BusinessEventLog
+from common.models.entitlements import BusinessEventLog
 from common.extensions import db
+from .celery_utils import current_celery
+from common.utils.prometheus_utils import sanitize_label
+
+# Standard duration buckets for all histograms
+DURATION_BUCKETS = [0.1, 0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 240, 360, float('inf')]
+
+# Prometheus metrics for business events
+TRACE_COUNTER = Counter(
+    'eveai_business_events_total',
+    'Total number of business events triggered',
+    ['tenant_id', 'event_type', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+TRACE_DURATION = Histogram(
+    'eveai_business_events_duration_seconds',
+    'Duration of business events in seconds',
+    ['tenant_id', 'event_type', 'specialist_id', 'specialist_type', 'specialist_type_version'],
+    buckets=DURATION_BUCKETS
+)
+
+CONCURRENT_TRACES = Gauge(
+    'eveai_business_events_concurrent',
+    'Number of concurrent business events',
+    ['tenant_id', 'event_type', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+SPAN_COUNTER = Counter(
+    'eveai_business_spans_total',
+    'Total number of spans within business events',
+    ['tenant_id', 'event_type', 'activity_name', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+SPAN_DURATION = Histogram(
+    'eveai_business_spans_duration_seconds',
+    'Duration of spans within business events in seconds',
+    ['tenant_id', 'event_type', 'activity_name', 'specialist_id', 'specialist_type', 'specialist_type_version'],
+    buckets=DURATION_BUCKETS
+)
+
+CONCURRENT_SPANS = Gauge(
+    'eveai_business_spans_concurrent',
+    'Number of concurrent spans within business events',
+    ['tenant_id', 'event_type', 'activity_name', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+# LLM Usage metrics
+LLM_TOKENS_COUNTER = Counter(
+    'eveai_llm_tokens_total',
+    'Total number of tokens used in LLM calls',
+    ['tenant_id', 'event_type', 'interaction_type', 'token_type', 'specialist_id', 'specialist_type',
+     'specialist_type_version']
+)
+
+LLM_DURATION = Histogram(
+    'eveai_llm_duration_seconds',
+    'Duration of LLM API calls in seconds',
+    ['tenant_id', 'event_type', 'interaction_type', 'specialist_id', 'specialist_type', 'specialist_type_version'],
+    buckets=DURATION_BUCKETS
+)
+
+LLM_CALLS_COUNTER = Counter(
+    'eveai_llm_calls_total',
+    'Total number of LLM API calls',
+    ['tenant_id', 'event_type', 'interaction_type', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
 
 
 class BusinessEvent:
@@ -25,18 +93,148 @@ class BusinessEvent:
         self.span_name = None
         self.parent_span_id = None
         self.document_version_id = kwargs.get('document_version_id')
+        self.document_version_file_size = kwargs.get('document_version_file_size')
         self.chat_session_id = kwargs.get('chat_session_id')
         self.interaction_id = kwargs.get('interaction_id')
+        self.specialist_id = kwargs.get('specialist_id')
+        self.specialist_type = kwargs.get('specialist_type')
+        self.specialist_type_version = kwargs.get('specialist_type_version')
         self.environment = os.environ.get("FLASK_ENV", "development")
         self.span_counter = 0
         self.spans = []
+        self.llm_metrics = {
+            'total_tokens': 0,
+            'prompt_tokens': 0,
+            'completion_tokens': 0,
+            'nr_of_pages': 0,
+            'total_time': 0,
+            'call_count': 0,
+            'interaction_type': None
+        }
+        self._log_buffer = []
+
+        # Prometheus label values must be strings
+        self.tenant_id_str = str(self.tenant_id)
+        self.event_type_str = sanitize_label(self.event_type)
+        self.specialist_id_str = str(self.specialist_id) if self.specialist_id else ""
+        self.specialist_type_str = str(self.specialist_type) if self.specialist_type else ""
+        self.specialist_type_version_str = sanitize_label(str(self.specialist_type_version)) \
+            if self.specialist_type_version else ""
+        self.span_name_str = ""
+
+        # Increment concurrent events gauge when initialized
+        CONCURRENT_TRACES.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        # Increment trace counter
+        TRACE_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        self._push_to_gateway()
 
     def update_attribute(self, attribute: str, value: any):
         if hasattr(self, attribute):
             setattr(self, attribute, value)
+            # Update string versions for Prometheus labels if needed
+            if attribute == 'specialist_id':
+                self.specialist_id_str = str(value) if value else ""
+            elif attribute == 'specialist_type':
+                self.specialist_type_str = str(value) if value else ""
+            elif attribute == 'specialist_type_version':
+                self.specialist_type_version_str = sanitize_label(str(value)) if value else ""
+            elif attribute == 'tenant_id':
+                self.tenant_id_str = str(value)
+            elif attribute == 'event_type':
+                self.event_type_str = sanitize_label(value)
+            elif attribute == 'span_name':
+                self.span_name_str = sanitize_label(value)
         else:
             raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'")
 
+    def update_llm_metrics(self, metrics: dict):
+        self.llm_metrics['total_tokens'] += metrics.get('total_tokens', 0)
+        self.llm_metrics['prompt_tokens'] += metrics.get('prompt_tokens', 0)
+        self.llm_metrics['completion_tokens'] += metrics.get('completion_tokens', 0)
+        self.llm_metrics['nr_of_pages'] += metrics.get('nr_of_pages', 0)
+        self.llm_metrics['total_time'] += metrics.get('time_elapsed', 0)
+        self.llm_metrics['call_count'] += 1
+        self.llm_metrics['interaction_type'] = metrics['interaction_type']
+
+        # Track in Prometheus metrics
+        interaction_type_str = sanitize_label(metrics['interaction_type']) if metrics['interaction_type'] else ""
+
+        # Track token usage
+        LLM_TOKENS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
+            token_type='total',
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc(metrics.get('total_tokens', 0))
+
+        LLM_TOKENS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
+            token_type='prompt',
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc(metrics.get('prompt_tokens', 0))
+
+        LLM_TOKENS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
+            token_type='completion',
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc(metrics.get('completion_tokens', 0))
+
+        # Track duration
+        LLM_DURATION.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).observe(metrics.get('time_elapsed', 0))
+
+        # Track call count
+        LLM_CALLS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            interaction_type=interaction_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        self._push_to_gateway()
+
+    def reset_llm_metrics(self):
+        self.llm_metrics['total_tokens'] = 0
+        self.llm_metrics['prompt_tokens'] = 0
+        self.llm_metrics['completion_tokens'] = 0
+        self.llm_metrics['nr_of_pages'] = 0
+        self.llm_metrics['total_time'] = 0
+        self.llm_metrics['call_count'] = 0
+        self.llm_metrics['interaction_type'] = None
+
     @contextmanager
     def create_span(self, span_name: str):
         # The create_span method is designed to be used as a context manager. We want to perform some actions when
@@ -54,25 +252,166 @@ class BusinessEvent:
         # Set the new span info
         self.span_id = new_span_id
         self.span_name = span_name
+        self.span_name_str = sanitize_label(span_name) if span_name else ""
         self.parent_span_id = parent_span_id
 
-        self.log(f"Starting span {span_name}")
+        # Track start time for the span
+        span_start_time = time.time()
+
+        # Increment span metrics - using span_name as activity_name for metrics
+        SPAN_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        # Increment concurrent spans gauge
+        CONCURRENT_SPANS.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        self._push_to_gateway()
+
+        self.log(f"Start")
 
         try:
             yield
         finally:
-            self.log(f"Ending span {span_name}")
+            # Calculate total time for this span
+            span_total_time = time.time() - span_start_time
+
+            # Observe span duration
+            SPAN_DURATION.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).observe(span_total_time)
+
+            # Decrement concurrent spans gauge
+            CONCURRENT_SPANS.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).dec()
+
+            self._push_to_gateway()
+
+            if self.llm_metrics['call_count'] > 0:
+                self.log_final_metrics()
+                self.reset_llm_metrics()
+            self.log(f"End", extra_fields={'span_duration': span_total_time})
             # Restore the previous span info
             if self.spans:
                 self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+                self.span_name_str = sanitize_label(span_name) if span_name else ""
             else:
                 self.span_id = None
                 self.span_name = None
                 self.parent_span_id = None
+                self.span_name_str = ""
 
-    def log(self, message: str, level: str = 'info'):
-        logger = logging.getLogger('business_events')
+    @asynccontextmanager
+    async def create_span_async(self, span_name: str):
+        """Async version of create_span using async context manager"""
+        parent_span_id = self.span_id
+        self.span_counter += 1
+        new_span_id = str(uuid.uuid4())
+
+        # Save the current span info
+        self.spans.append((self.span_id, self.span_name, self.parent_span_id))
+
+        # Set the new span info
+        self.span_id = new_span_id
+        self.span_name = span_name
+        self.span_name_str = sanitize_label(span_name) if span_name else ""
+        self.parent_span_id = parent_span_id
+
+        # Track start time for the span
+        span_start_time = time.time()
+
+        # Increment span metrics - using span_name as activity_name for metrics
+        SPAN_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        # Increment concurrent spans gauge
+        CONCURRENT_SPANS.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            activity_name=self.span_name_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        self._push_to_gateway()
+
+        self.log(f"Start")
+
+        try:
+            yield
+        finally:
+            # Calculate total time for this span
+            span_total_time = time.time() - span_start_time
+
+            # Observe span duration
+            SPAN_DURATION.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).observe(span_total_time)
+
+            # Decrement concurrent spans gauge
+            CONCURRENT_SPANS.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type_str,
+                activity_name=self.span_name_str,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).dec()
+
+            self._push_to_gateway()
+
+            if self.llm_metrics['call_count'] > 0:
+                self.log_final_metrics()
+                self.reset_llm_metrics()
+            self.log(f"End", extra_fields={'span_duration': span_total_time})
+            # Restore the previous span info
+            if self.spans:
+                self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+                self.span_name_str = sanitize_label(span_name) if span_name else ""
+            else:
+                self.span_id = None
+                self.span_name = None
+                self.parent_span_id = None
+                self.span_name_str = ""
+
+    def log(self, message: str, level: str = 'info', extra_fields: Dict[str, Any] = None):
         log_data = {
+            'timestamp': dt.now(tz=tz.utc),
             'event_type': self.event_type,
             'tenant_id': self.tenant_id,
             'trace_id': self.trace_id,
@@ -80,35 +419,226 @@ class BusinessEvent:
             'span_name': self.span_name,
             'parent_span_id': self.parent_span_id,
             'document_version_id': self.document_version_id,
+            'document_version_file_size': self.document_version_file_size,
             'chat_session_id': self.chat_session_id,
             'interaction_id': self.interaction_id,
-            'environment': self.environment
+            'specialist_id': self.specialist_id,
+            'specialist_type': self.specialist_type,
+            'specialist_type_version': self.specialist_type_version,
+            'environment': self.environment,
+            'message': message,
         }
-        # log to Graylog
-        getattr(logger, level)(message, extra=log_data)
+        # Add any extra fields
+        if extra_fields:
+            for key, value in extra_fields.items():
+                # For span/trace duration, use the llm_metrics_total_time field
+                if key == 'span_duration' or key == 'trace_duration':
+                    log_data['llm_metrics_total_time'] = value
+                else:
+                    log_data[key] = value
 
-        # Log to database
-        event_log = BusinessEventLog(
-            timestamp=dt.now(tz=tz.utc),
-            event_type=self.event_type,
-            tenant_id=self.tenant_id,
-            trace_id=self.trace_id,
-            span_id=self.span_id,
-            span_name=self.span_name,
-            parent_span_id=self.parent_span_id,
-            document_version_id=self.document_version_id,
-            chat_session_id=self.chat_session_id,
-            interaction_id=self.interaction_id,
-            environment=self.environment,
-            message=message
-        )
-        db.session.add(event_log)
-        db.session.commit()
+        self._log_buffer.append(log_data)
+
+    def log_llm_metrics(self, metrics: dict, level: str = 'info'):
+        self.update_llm_metrics(metrics)
+        message = "LLM Metrics"
+        logger = logging.getLogger('business_events')
+        log_data = {
+            'timestamp': dt.now(tz=tz.utc),
+            'event_type': self.event_type,
+            'tenant_id': self.tenant_id,
+            'trace_id': self.trace_id,
+            'span_id': self.span_id,
+            'span_name': self.span_name,
+            'parent_span_id': self.parent_span_id,
+            'document_version_id': self.document_version_id,
+            'document_version_file_size': self.document_version_file_size,
+            'chat_session_id': self.chat_session_id,
+            'interaction_id': self.interaction_id,
+            'specialist_id': self.specialist_id,
+            'specialist_type': self.specialist_type,
+            'specialist_type_version': self.specialist_type_version,
+            'environment': self.environment,
+            'llm_metrics_total_tokens': metrics.get('total_tokens', 0),
+            'llm_metrics_prompt_tokens': metrics.get('prompt_tokens', 0),
+            'llm_metrics_completion_tokens': metrics.get('completion_tokens', 0),
+            'llm_metrics_nr_of_pages': metrics.get('nr_of_pages', 0),
+            'llm_metrics_total_time': metrics.get('time_elapsed', 0),
+            'llm_interaction_type': metrics['interaction_type'],
+            'message': message,
+        }
+        self._log_buffer.append(log_data)
+
+    def log_final_metrics(self, level: str = 'info'):
+        logger = logging.getLogger('business_events')
+        message = "Final LLM Metrics"
+        log_data = {
+            'timestamp': dt.now(tz=tz.utc),
+            'event_type': self.event_type,
+            'tenant_id': self.tenant_id,
+            'trace_id': self.trace_id,
+            'span_id': self.span_id,
+            'span_name': self.span_name,
+            'parent_span_id': self.parent_span_id,
+            'document_version_id': self.document_version_id,
+            'document_version_file_size': self.document_version_file_size,
+            'chat_session_id': self.chat_session_id,
+            'interaction_id': self.interaction_id,
+            'specialist_id': self.specialist_id,
+            'specialist_type': self.specialist_type,
+            'specialist_type_version': self.specialist_type_version,
+            'environment': self.environment,
+            'llm_metrics_total_tokens': self.llm_metrics['total_tokens'],
+            'llm_metrics_prompt_tokens': self.llm_metrics['prompt_tokens'],
+            'llm_metrics_completion_tokens': self.llm_metrics['completion_tokens'],
+            'llm_metrics_nr_of_pages': self.llm_metrics['nr_of_pages'],
+            'llm_metrics_total_time': self.llm_metrics['total_time'],
+            'llm_metrics_call_count': self.llm_metrics['call_count'],
+            'llm_interaction_type': self.llm_metrics['interaction_type'],
+            'message': message,
+        }
+        self._log_buffer.append(log_data)
+
+    @staticmethod
+    def _direct_db_persist(log_entries: List[Dict[str, Any]]):
+        """Fallback method to directly persist logs to DB if async fails"""
+        try:
+            db_entries = []
+            for entry in log_entries:
+                event_log = BusinessEventLog(
+                    timestamp=entry.pop('timestamp'),
+                    event_type=entry.pop('event_type'),
+                    tenant_id=entry.pop('tenant_id'),
+                    trace_id=entry.pop('trace_id'),
+                    span_id=entry.pop('span_id', None),
+                    span_name=entry.pop('span_name', None),
+                    parent_span_id=entry.pop('parent_span_id', None),
+                    document_version_id=entry.pop('document_version_id', None),
+                    document_version_file_size=entry.pop('document_version_file_size', None),
+                    chat_session_id=entry.pop('chat_session_id', None),
+                    interaction_id=entry.pop('interaction_id', None),
+                    specialist_id=entry.pop('specialist_id', None),
+                    specialist_type=entry.pop('specialist_type', None),
+                    specialist_type_version=entry.pop('specialist_type_version', None),
+                    environment=entry.pop('environment', None),
+                    llm_metrics_total_tokens=entry.pop('llm_metrics_total_tokens', None),
+                    llm_metrics_prompt_tokens=entry.pop('llm_metrics_prompt_tokens', None),
+                    llm_metrics_completion_tokens=entry.pop('llm_metrics_completion_tokens', None),
+                    llm_metrics_total_time=entry.pop('llm_metrics_total_time', None),
+                    llm_metrics_call_count=entry.pop('llm_metrics_call_count', None),
+                    llm_interaction_type=entry.pop('llm_interaction_type', None),
+                    message=entry.pop('message', None)
+                )
+                db_entries.append(event_log)
+
+            # Bulk insert
+            db.session.bulk_save_objects(db_entries)
+            db.session.commit()
+        except Exception as e:
+            logger = logging.getLogger('business_events')
+            logger.error(f"Failed to persist logs directly to DB: {e}")
+            db.session.rollback()
+
+    def _flush_log_buffer(self):
+        """Flush the log buffer to the database via a Celery task"""
+        if self._log_buffer:
+            try:
+                # Send to Celery task
+                current_celery.send_task(
+                    'persist_business_events',
+                    args=[self._log_buffer],
+                    queue='entitlements'  # Or dedicated log queue
+                )
+            except Exception as e:
+                # Fallback to direct DB write in case of issues with Celery
+                logger = logging.getLogger('business_events')
+                logger.error(f"Failed to send logs to Celery. Falling back to direct DB: {e}")
+                self._direct_db_persist(self._log_buffer)
+
+            # Clear the buffer after sending
+            self._log_buffer = []
+
+    def _push_to_gateway(self):
+        # Push metrics to the gateway
+        try:
+            push_to_gateway(
+                current_app.config['PUSH_GATEWAY_URL'],
+                job=current_app.config['COMPONENT_NAME'],
+                registry=REGISTRY
+            )
+        except Exception as e:
+            current_app.logger.error(f"Failed to push metrics to Prometheus Push Gateway: {e}")
 
     def __enter__(self):
+        self.trace_start_time = time.time()
         self.log(f'Starting Trace for {self.event_type}')
         return BusinessEventContext(self).__enter__()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.log(f'Ending Trace for {self.event_type}')
+        trace_total_time = time.time() - self.trace_start_time
+
+        # Record trace duration
+        TRACE_DURATION.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).observe(trace_total_time)
+
+        # Decrement concurrent traces gauge
+        CONCURRENT_TRACES.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).dec()
+
+        self._push_to_gateway()
+
+        if self.llm_metrics['call_count'] > 0:
+            self.log_final_metrics()
+            self.reset_llm_metrics()
+
+        self.log(f'Ending Trace for {self.event_type}', extra_fields={'trace_duration': trace_total_time})
+        self._flush_log_buffer()
+
+
         return BusinessEventContext(self).__exit__(exc_type, exc_val, exc_tb)
+
+    async def __aenter__(self):
+        self.trace_start_time = time.time()
+        self.log(f'Starting Trace for {self.event_type}')
+        return await BusinessEventContext(self).__aenter__()
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        trace_total_time = time.time() - self.trace_start_time
+
+        # Record trace duration
+        TRACE_DURATION.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).observe(trace_total_time)
+
+        # Decrement concurrent traces gauge
+        CONCURRENT_TRACES.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type_str,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).dec()
+
+        self._push_to_gateway()
+
+        if self.llm_metrics['call_count'] > 0:
+            self.log_final_metrics()
+            self.reset_llm_metrics()
+
+        self.log(f'Ending Trace for {self.event_type}', extra_fields={'trace_duration': trace_total_time})
+        self._flush_log_buffer()
+        return await BusinessEventContext(self).__aexit__(exc_type, exc_val, exc_tb)

common/utils/business_event_context.py

@@ -1,9 +1,22 @@
 from werkzeug.local import LocalProxy, LocalStack
+import asyncio
+from contextvars import ContextVar
+import contextvars
 
+# Keep existing stack for backward compatibility
 _business_event_stack = LocalStack()
 
+# Add contextvar for async support
+_business_event_contextvar = ContextVar('business_event', default=None)
+
 
 def _get_current_event():
+    # Try contextvar first (for async)
+    event = _business_event_contextvar.get()
+    if event is not None:
+        return event
+
+    # Fall back to the stack-based approach (for sync)
     top = _business_event_stack.top
     if top is None:
         raise RuntimeError("No business event context found. Are you sure you're in a business event?")
@@ -16,10 +29,24 @@ current_event = LocalProxy(_get_current_event)
 class BusinessEventContext:
     def __init__(self, event):
         self.event = event
+        self._token = None  # For storing contextvar token
 
     def __enter__(self):
         _business_event_stack.push(self.event)
+        self._token = _business_event_contextvar.set(self.event)
         return self.event
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         _business_event_stack.pop()
+        if self._token is not None:
+            _business_event_contextvar.reset(self._token)
+
+    async def __aenter__(self):
+        _business_event_stack.push(self.event)
+        self._token = _business_event_contextvar.set(self.event)
+        return self.event
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        _business_event_stack.pop()
+        if self._token is not None:
+            _business_event_contextvar.reset(self._token)

common/utils/cache/base.py

@@ -0,0 +1,192 @@
+from typing import Any, Dict, List, Optional, TypeVar, Generic, Type
+from dataclasses import dataclass
+from flask import Flask, current_app
+from dogpile.cache import CacheRegion
+from abc import ABC, abstractmethod
+
+T = TypeVar('T')  # Generic type parameter for cached data
+
+
+@dataclass
+class CacheKey:
+    """
+    Represents a composite cache key made up of multiple components.
+    Enables structured and consistent key generation for cache entries.
+
+    Attributes:
+        components (Dict[str, Any]): Dictionary of key components and their values
+
+    Example:
+        key = CacheKey({'tenant_id': 123, 'user_id': 456})
+        str(key) -> "tenant_id=123:user_id=456"
+    """
+    components: Dict[str, Any]
+
+    def __str__(self) -> str:
+        """
+        Converts components into a deterministic string representation.
+        Components are sorted alphabetically to ensure consistent key generation.
+        """
+        return ":".join(f"{k}={v}" for k, v in sorted(self.components.items()))
+
+
+class CacheHandler(Generic[T]):
+    """
+    Base cache handler implementation providing structured caching functionality.
+    Uses generics to ensure type safety of cached data.
+
+    Type Parameters:
+        T: Type of data being cached
+
+    Attributes:
+        region (CacheRegion): Dogpile cache region for storage
+        prefix (str): Prefix for all cache keys managed by this handler
+    """
+
+    def __init__(self, region: CacheRegion, prefix: str):
+        self.region = region
+        self.prefix = prefix
+        self._key_components = []  # List of required key components
+
+    @abstractmethod
+    def _to_cache_data(self, instance: T) -> Any:
+        """
+        Convert the data to a cacheable format for internal use.
+
+        Args:
+            instance: The data to be cached.
+
+        Returns:
+            A serializable format of the instance.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _from_cache_data(self, data: Any, **kwargs) -> T:
+        """
+        Convert cached data back to usable format for internal use.
+
+        Args:
+            data: The cached data.
+            **kwargs: Additional context.
+
+        Returns:
+            The data in its usable format.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _should_cache(self, value: T) -> bool:
+        """
+        Validate if the value should be cached for internal use.
+
+        Args:
+            value: The value to be cached.
+
+        Returns:
+            True if the value should be cached, False otherwise.
+        """
+        raise NotImplementedError
+
+    def configure_keys(self, *components: str):
+        """
+        Configure required components for cache key generation.
+
+        Args:
+            *components: Required key component names
+
+        Returns:
+            self for method chaining
+        """
+        self._key_components = components
+        return self
+
+    def generate_key(self, **identifiers) -> str:
+        """
+        Generate a cache key from provided identifiers.
+
+        Args:
+            **identifiers: Key-value pairs for key components
+
+        Returns:
+            Formatted cache key string
+
+        Raises:
+            ValueError: If required components are missing
+        """
+        missing = set(self._key_components) - set(identifiers.keys())
+        if missing:
+            raise ValueError(f"Missing key components: {missing}")
+
+        region_name = getattr(self.region, 'name', 'default_region')
+
+        key = CacheKey({k: identifiers[k] for k in self._key_components})
+        return f"{region_name}_{self.prefix}:{str(key)}"
+
+    def get(self, creator_func, **identifiers) -> T:
+        """
+        Get or create a cached value.
+
+        Args:
+            creator_func: Function to create value if not cached
+            **identifiers: Key components for cache key
+
+        Returns:
+            Cached or newly created value
+        """
+        cache_key = self.generate_key(**identifiers)
+
+        def creator():
+            instance = creator_func(**identifiers)
+            serialized_instance = self._to_cache_data(instance)
+            return serialized_instance
+
+        cached_data = self.region.get_or_create(
+            cache_key,
+            creator,
+            should_cache_fn=self._should_cache
+        )
+
+        return self._from_cache_data(cached_data, **identifiers)
+
+    def invalidate(self, **identifiers):
+        """
+        Invalidate a specific cache entry.
+
+        Args:
+            **identifiers: Key components for the cache entry
+        """
+        cache_key = self.generate_key(**identifiers)
+        self.region.delete(cache_key)
+
+    def invalidate_by_model(self, model: str, **identifiers):
+        """
+        Invalidate cache entry based on model changes.
+
+        Args:
+            model: Changed model name
+            **identifiers: Model instance identifiers
+        """
+        try:
+            self.invalidate(**identifiers)
+        except ValueError:
+            pass  # Skip if cache key can't be generated from provided identifiers
+
+    def invalidate_region(self):
+        """
+        Invalidate all cache entries within this region.
+
+        Deletes all keys that start with the region prefix.
+        """
+        # Construct the pattern for all keys in this region
+        pattern = f"{self.region}_{self.prefix}:*"
+
+        # Assuming Redis backend with dogpile, use `delete_multi` or direct Redis access
+        if hasattr(self.region.backend, 'client'):
+            redis_client = self.region.backend.client
+            keys_to_delete = redis_client.keys(pattern)
+            if keys_to_delete:
+                redis_client.delete(*keys_to_delete)
+        else:
+            # Fallback for other backends
+            raise NotImplementedError("Region invalidation is only supported for Redis backend.")

common/utils/cache/config_cache.py

@@ -0,0 +1,526 @@
+from typing import Dict, Any, Optional
+from pathlib import Path
+import yaml
+from packaging import version
+import os
+from flask import current_app
+
+from common.utils.cache.base import CacheHandler, CacheKey
+from config.type_defs import agent_types, task_types, tool_types, specialist_types, retriever_types, prompt_types, \
+    catalog_types, partner_service_types, processor_types, customisation_types
+
+
+def is_major_minor(version: str) -> bool:
+    parts = version.strip('.').split('.')
+    return len(parts) == 2 and all(part.isdigit() for part in parts)
+
+
+class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
+    """Base handler for configuration caching"""
+
+    def __init__(self, region, config_type: str):
+        """
+        Args:
+            region: Cache region
+            config_type: Type of configuration (agents, tasks, etc.)
+        """
+        super().__init__(region, f'config_{config_type}')
+        self.config_type = config_type
+        self._types_module = None  # Set by subclasses
+        self._config_dir = None  # Set by subclasses
+        self.version_tree_cache = None
+        self.configure_keys('type_name', 'version')
+
+    def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert the data to a cacheable format"""
+        # For configuration data, we can just return the dictionary as is
+        # since it's already in a serializable format
+        return instance
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Convert cached data back to usable format"""
+        # Similarly, we can return the data directly since it's already
+        # in the format we need
+        return data
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """
+        Validate if the value should be cached
+
+        Args:
+            value: The value to be cached
+
+        Returns:
+            bool: True if the value should be cached
+        """
+        return isinstance(value, dict)  # Cache all dictionaries
+
+    def set_version_tree_cache(self, cache):
+        """Set the version tree cache dependency."""
+        self.version_tree_cache = cache
+
+    def _load_specific_config(self, type_name: str, version_str: str = 'latest') -> Dict[str, Any]:
+        """
+        Load a specific configuration version
+        Automatically handles global vs partner-specific configs
+        """
+        version_tree = self.version_tree_cache.get_versions(type_name)
+        versions = version_tree['versions']
+
+        if version_str == 'latest':
+            version_str = version_tree['latest_version']
+
+        if version_str not in versions:
+            raise ValueError(f"Version {version_str} not found for {type_name}")
+
+        version_info = versions[version_str]
+        file_path = version_info['file_path']
+        partner = version_info.get('partner')
+
+        try:
+            with open(file_path) as f:
+                config = yaml.safe_load(f)
+                # Add partner information to the config
+                if partner:
+                    config['partner'] = partner
+                return config
+        except Exception as e:
+            raise ValueError(f"Error loading config from {file_path}: {e}")
+
+    def get_config(self, type_name: str, version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get configuration for a specific type and version
+        If version not specified, returns latest
+
+        Args:
+            type_name: Configuration type name
+            version: Optional specific version to retrieve
+
+        Returns:
+            Configuration data
+        """
+        if version is None:
+            version_str = self.version_tree_cache.get_latest_version(type_name)
+        elif is_major_minor(version):
+            version_str = self.version_tree_cache.get_latest_patch_version(type_name, version)
+        else:
+            version_str = version
+
+        result = self.get(
+            lambda type_name, version: self._load_specific_config(type_name, version),
+            type_name=type_name,
+            version=version_str
+        )
+        return result
+
+
+class BaseConfigVersionTreeCacheHandler(CacheHandler[Dict[str, Any]]):
+    """Base handler for configuration version tree caching"""
+
+    def __init__(self, region, config_type: str):
+        """
+        Args:
+            region: Cache region
+            config_type: Type of configuration (agents, tasks, etc.)
+        """
+        super().__init__(region, f'config_{config_type}_version_tree')
+        self.config_type = config_type
+        self._types_module = None  # Set by subclasses
+        self._config_dir = None  # Set by subclasses
+        self.configure_keys('type_name')
+
+    def _load_version_tree(self, type_name: str) -> Dict[str, Any]:
+        """
+        Load version tree for a specific type without loading full configurations
+        Checks both global and partner-specific directories
+        """
+        # First check the global path
+        global_path = Path(self._config_dir) / "globals" / type_name
+
+        # If global path doesn't exist, check if the type exists directly in the root
+        # (for backward compatibility)
+        if not global_path.exists():
+            global_path = Path(self._config_dir) / type_name
+
+        if not global_path.exists():
+            # Check if it exists in any partner subdirectories
+            partner_dirs = [d for d in Path(self._config_dir).iterdir()
+                            if d.is_dir() and d.name != "globals"]
+
+            for partner_dir in partner_dirs:
+                partner_type_path = partner_dir / type_name
+                if partner_type_path.exists():
+                    # Found in partner directory
+                    return self._load_versions_from_path(partner_type_path)
+
+            # If we get here, the type wasn't found anywhere
+            raise ValueError(f"No configuration found for type {type_name}")
+
+        return self._load_versions_from_path(global_path)
+
+    def _load_versions_from_path(self, path: Path) -> Dict[str, Any]:
+        """Load all versions from a specific path"""
+        version_files = list(path.glob('*.yaml'))
+        if not version_files:
+            raise ValueError(f"No versions found in {path}")
+
+        versions = {}
+        latest_version = None
+        latest_version_obj = None
+
+        for file_path in version_files:
+            ver = file_path.stem  # Get version from filename
+            try:
+                ver_obj = version.parse(ver)
+                # Only load minimal metadata for version tree
+                with open(file_path) as f:
+                    yaml_data = yaml.safe_load(f)
+                    metadata = yaml_data.get('metadata', {})
+                    # Add partner information if available
+                    partner = None
+                    if "globals" not in str(file_path):
+                        # Extract partner name from path
+                        # Path format: config_dir/partner_name/type_name/version.yaml
+                        partner = file_path.parent.parent.name
+
+                    versions[ver] = {
+                        'metadata': metadata,
+                        'file_path': str(file_path),
+                        'partner': partner
+                    }
+
+                # Track latest version
+                if latest_version_obj is None or ver_obj > latest_version_obj:
+                    latest_version = ver
+                    latest_version_obj = ver_obj
+
+            except Exception as e:
+                current_app.logger.error(f"Error loading version {ver}: {e}")
+                continue
+
+        return {
+            'versions': versions,
+            'latest_version': latest_version
+        }
+
+    def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert the data to a cacheable format"""
+        # For configuration data, we can just return the dictionary as is
+        # since it's already in a serializable format
+        return instance
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Convert cached data back to usable format"""
+        # Similarly, we can return the data directly since it's already
+        # in the format we need
+        return data
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """
+        Validate if the value should be cached
+
+        Args:
+            value: The value to be cached
+
+        Returns:
+            bool: True if the value should be cached
+        """
+        return isinstance(value, dict)  # Cache all dictionaries
+
+    def get_versions(self, type_name: str) -> Dict[str, Any]:
+        """
+        Get version tree for a type
+
+        Args:
+            type_name: Type to get versions for
+
+        Returns:
+            Dict with version information
+        """
+        return self.get(
+            lambda type_name: self._load_version_tree(type_name),
+            type_name=type_name
+        )
+
+    def get_latest_version(self, type_name: str) -> str:
+        """
+        Get the latest version for a given type name.
+
+        Args:
+            type_name: Name of the configuration type
+
+        Returns:
+            Latest version string
+
+        Raises:
+            ValueError: If type not found or no versions available
+        """
+        version_tree = self.get_versions(type_name)
+        if not version_tree or 'latest_version' not in version_tree:
+            raise ValueError(f"No versions found for {type_name}")
+
+        return version_tree['latest_version']
+
+    def get_latest_patch_version(self, type_name: str, major_minor: str) -> str:
+        """
+        Get the latest patch version for a given major.minor version.
+
+        Args:
+            type_name: Name of the configuration type
+            major_minor: Major.minor version (e.g. "1.0")
+
+        Returns:
+            Latest patch version string (e.g. "1.0.3")
+
+        Raises:
+            ValueError: If type not found or no matching versions
+        """
+        version_tree = self.get_versions(type_name)
+        if not version_tree or 'versions' not in version_tree:
+            raise ValueError(f"No versions found for {type_name}")
+
+        # Filter versions that match the major.minor prefix
+        matching_versions = [
+            ver for ver in version_tree['versions'].keys()
+            if ver.startswith(major_minor + '.')
+        ]
+
+        if not matching_versions:
+            raise ValueError(f"No versions found for {type_name} with prefix {major_minor}")
+
+        # Return highest matching version
+        latest_patch = max(matching_versions, key=version.parse)
+        return latest_patch
+
+
+class BaseConfigTypesCacheHandler(CacheHandler[Dict[str, Any]]):
+    """Base handler for configuration types caching"""
+
+    def __init__(self, region, config_type: str):
+        """
+        Args:
+            region: Cache region
+            config_type: Type of configuration (agents, tasks, etc.)
+        """
+        super().__init__(region, f'config_{config_type}_types')
+        self.config_type = config_type
+        self._types_module = None  # Set by subclasses
+        self._config_dir = None  # Set by subclasses
+        self.configure_keys()
+
+    def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert the data to a cacheable format"""
+        # For configuration data, we can just return the dictionary as is
+        # since it's already in a serializable format
+        return instance
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Convert cached data back to usable format"""
+        # Similarly, we can return the data directly since it's already
+        # in the format we need
+        return data
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """
+        Validate if the value should be cached
+
+        Args:
+            value: The value to be cached
+
+        Returns:
+            bool: True if the value should be cached
+        """
+        return isinstance(value, dict)  # Cache all dictionaries
+
+    def _load_type_definitions(self) -> Dict[str, Dict[str, str]]:
+        """Load type definitions from the corresponding type_defs module"""
+        if not self._types_module:
+            raise ValueError("_types_module must be set by subclass")
+
+        type_definitions = {
+            type_id: {
+                'name': info['name'],
+                'description': info['description'],
+                'partner': info.get('partner')  # Include partner info if available
+            }
+            for type_id, info in self._types_module.items()
+        }
+
+        return type_definitions
+
+    def get_types(self) -> Dict[str, Dict[str, str]]:
+        """Get dictionary of available types with name and description"""
+        result = self.get(
+            lambda type_name: self._load_type_definitions(),
+            type_name=f'{self.config_type}_types',
+        )
+        return result
+
+
+def create_config_cache_handlers(config_type: str, config_dir: str, types_module: dict) -> tuple:
+    """
+    Factory function to dynamically create the 3 cache handler classes for a given configuration type.
+    The following cache names are created:
+    - <config_type>_config_cache
+    - <config_type>_version_tree_cache
+    - <config_type>_types_cache
+
+
+    Args:
+        config_type: The configuration type (e.g., 'agents', 'tasks').
+        config_dir: The directory where configuration files are stored.
+        types_module: The types module defining the available types for this config.
+
+    Returns:
+        A tuple of dynamically created classes for config, version tree, and types handlers.
+    """
+
+    class ConfigCacheHandler(BaseConfigCacheHandler):
+        handler_name = f"{config_type}_config_cache"
+
+        def __init__(self, region):
+            super().__init__(region, config_type)
+            self._types_module = types_module
+            self._config_dir = config_dir
+
+    class VersionTreeCacheHandler(BaseConfigVersionTreeCacheHandler):
+        handler_name = f"{config_type}_version_tree_cache"
+
+        def __init__(self, region):
+            super().__init__(region, config_type)
+            self._types_module = types_module
+            self._config_dir = config_dir
+
+    class TypesCacheHandler(BaseConfigTypesCacheHandler):
+        handler_name = f"{config_type}_types_cache"
+
+        def __init__(self, region):
+            super().__init__(region, config_type)
+            self._types_module = types_module
+            self._config_dir = config_dir
+
+    return ConfigCacheHandler, VersionTreeCacheHandler, TypesCacheHandler
+
+
+AgentConfigCacheHandler, AgentConfigVersionTreeCacheHandler, AgentConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='agents',
+        config_dir='config/agents',
+        types_module=agent_types.AGENT_TYPES
+    ))
+
+
+TaskConfigCacheHandler, TaskConfigVersionTreeCacheHandler, TaskConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='tasks',
+        config_dir='config/tasks',
+        types_module=task_types.TASK_TYPES
+    ))
+
+
+ToolConfigCacheHandler, ToolConfigVersionTreeCacheHandler, ToolConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='tools',
+        config_dir='config/tools',
+        types_module=tool_types.TOOL_TYPES
+    ))
+
+
+SpecialistConfigCacheHandler, SpecialistConfigVersionTreeCacheHandler, SpecialistConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='specialists',
+        config_dir='config/specialists',
+        types_module=specialist_types.SPECIALIST_TYPES
+    ))
+
+
+RetrieverConfigCacheHandler, RetrieverConfigVersionTreeCacheHandler, RetrieverConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='retrievers',
+        config_dir='config/retrievers',
+        types_module=retriever_types.RETRIEVER_TYPES
+    ))
+
+
+PromptConfigCacheHandler, PromptConfigVersionTreeCacheHandler, PromptConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='prompts',
+        config_dir='config/prompts',
+        types_module=prompt_types.PROMPT_TYPES
+    ))
+
+CatalogConfigCacheHandler, CatalogConfigVersionTreeCacheHandler, CatalogConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='catalogs',
+        config_dir='config/catalogs',
+        types_module=catalog_types.CATALOG_TYPES
+    ))
+
+ProcessorConfigCacheHandler, ProcessorConfigVersionTreeCacheHandler, ProcessorConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='processors',
+        config_dir='config/processors',
+        types_module=processor_types.PROCESSOR_TYPES
+    ))
+
+PartnerServiceConfigCacheHandler, PartnerServiceConfigVersionTreeCacheHandler, PartnerServiceConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='partner_services',
+        config_dir='config/partner_services',
+        types_module=partner_service_types.PARTNER_SERVICE_TYPES
+    ))
+
+CustomisationConfigCacheHandler, CustomisationConfigVersionTreeCacheHandler, CustomisationConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='customisations',
+        config_dir='config/customisations',
+        types_module=customisation_types.CUSTOMISATION_TYPES
+    )
+)
+
+
+def register_config_cache_handlers(cache_manager) -> None:
+    cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(TaskConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(TaskConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(TaskConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ToolConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ToolConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ToolConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(RetrieverConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(RetrieverConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(RetrieverConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PromptConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PromptConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PromptConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CatalogConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CatalogConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CatalogConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PartnerServiceConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PartnerServiceConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PartnerServiceConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CustomisationConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CustomisationConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CustomisationConfigVersionTreeCacheHandler, 'eveai_config')
+
+    cache_manager.agents_config_cache.set_version_tree_cache(cache_manager.agents_version_tree_cache)
+    cache_manager.tasks_config_cache.set_version_tree_cache(cache_manager.tasks_version_tree_cache)
+    cache_manager.tools_config_cache.set_version_tree_cache(cache_manager.tools_version_tree_cache)
+    cache_manager.specialists_config_cache.set_version_tree_cache(cache_manager.specialists_version_tree_cache)
+    cache_manager.retrievers_config_cache.set_version_tree_cache(cache_manager.retrievers_version_tree_cache)
+    cache_manager.prompts_config_cache.set_version_tree_cache(cache_manager.prompts_version_tree_cache)
+    cache_manager.catalogs_config_cache.set_version_tree_cache(cache_manager.catalogs_version_tree_cache)
+    cache_manager.processors_config_cache.set_version_tree_cache(cache_manager.processors_version_tree_cache)
+    cache_manager.partner_services_config_cache.set_version_tree_cache(cache_manager.partner_services_version_tree_cache)
+    cache_manager.customisations_config_cache.set_version_tree_cache(cache_manager.customisations_version_tree_cache)

common/utils/cache/crewai_config_processor.py

@@ -0,0 +1,218 @@
+from typing import Dict, Any, Type, TypeVar, List
+from abc import ABC, abstractmethod
+from flask import current_app
+
+from common.extensions import cache_manager, db
+from common.models.interaction import EveAIAgent, EveAITask, EveAITool, Specialist
+from common.utils.cache.crewai_configuration import (
+    ProcessedAgentConfig, ProcessedTaskConfig, ProcessedToolConfig,
+    SpecialistProcessedConfig
+)
+
+T = TypeVar('T')  # For generic model types
+
+
+class BaseCrewAIConfigProcessor:
+    """Base processor for specialist configurations"""
+
+    # Standard mapping between model fields and template placeholders
+    AGENT_FIELD_MAPPING = {
+        'role': 'custom_role',
+        'goal': 'custom_goal',
+        'backstory': 'custom_backstory'
+    }
+
+    TASK_FIELD_MAPPING = {
+        'task_description': 'custom_description',
+        'expected_output': 'custom_expected_output'
+    }
+
+    def __init__(self, tenant_id: int, specialist_id: int):
+        self.tenant_id = tenant_id
+        self.specialist_id = specialist_id
+        self.specialist = self._get_specialist()
+        self.verbose = self._get_verbose_setting()
+
+    def _get_specialist(self) -> Specialist:
+        """Get specialist and verify existence"""
+        specialist = Specialist.query.get(self.specialist_id)
+        if not specialist:
+            raise ValueError(f"Specialist {self.specialist_id} not found")
+        return specialist
+
+    def _get_verbose_setting(self) -> bool:
+        """Get verbose setting from specialist"""
+        return bool(self.specialist.tuning)
+
+    def _get_db_items(self, model_class: Type[T], type_list: List[str]) -> Dict[str, T]:
+        """Get database items of specified type"""
+        items = (model_class.query
+                 .filter_by(specialist_id=self.specialist_id)
+                 .filter(model_class.type.in_(type_list))
+                 .all())
+        return {item.type: item for item in items}
+
+    def _apply_replacements(self, text: str, replacements: Dict[str, str]) -> str:
+        """Apply text replacements to a string"""
+        result = text
+        for key, value in replacements.items():
+            if value is not None:  # Only replace if value exists
+                placeholder = "{" + key + "}"
+                result = result.replace(placeholder, str(value))
+        return result
+
+    def _process_agent_configs(self, specialist_config: Dict[str, Any]) -> Dict[str, ProcessedAgentConfig]:
+        """Process all agent configurations"""
+        agent_configs = {}
+
+        if 'agents' not in specialist_config:
+            return agent_configs
+
+        # Get all DB agents at once
+        agent_types = [agent_def['type'] for agent_def in specialist_config['agents']]
+        db_agents = self._get_db_items(EveAIAgent, agent_types)
+
+        for agent_def in specialist_config['agents']:
+            agent_type = agent_def['type']
+            agent_type_lower = agent_type.lower()
+            db_agent = db_agents.get(agent_type)
+
+            # Get full configuration
+            config = cache_manager.agents_config_cache.get_config(
+                agent_type,
+                agent_def.get('version', '1.0')
+            )
+
+            # Start with YAML values
+            role = config['role']
+            goal = config['goal']
+            backstory = config['backstory']
+
+            # Apply DB values if they exist
+            if db_agent:
+                for model_field, placeholder in self.AGENT_FIELD_MAPPING.items():
+                    value = getattr(db_agent, model_field)
+                    if value:
+                        placeholder_text = "{" + placeholder + "}"
+                        role = role.replace(placeholder_text, value)
+                        goal = goal.replace(placeholder_text, value)
+                        backstory = backstory.replace(placeholder_text, value)
+
+            agent_configs[agent_type_lower] = ProcessedAgentConfig(
+                role=role,
+                goal=goal,
+                backstory=backstory,
+                name=agent_def.get('name') or config.get('name', agent_type_lower),
+                type=agent_type,
+                description=agent_def.get('description') or config.get('description'),
+                verbose=self.verbose
+            )
+
+        return agent_configs
+
+    def _process_task_configs(self, specialist_config: Dict[str, Any]) -> Dict[str, ProcessedTaskConfig]:
+        """Process all task configurations"""
+        task_configs = {}
+
+        if 'tasks' not in specialist_config:
+            return task_configs
+
+        # Get all DB tasks at once
+        task_types = [task_def['type'] for task_def in specialist_config['tasks']]
+        db_tasks = self._get_db_items(EveAITask, task_types)
+
+        for task_def in specialist_config['tasks']:
+            task_type = task_def['type']
+            task_type_lower = task_type.lower()
+            db_task = db_tasks.get(task_type)
+
+            # Get full configuration
+            config = cache_manager.tasks_config_cache.get_config(
+                task_type,
+                task_def.get('version', '1.0')
+            )
+
+            # Start with YAML values
+            task_description = config['task_description']
+            expected_output = config['expected_output']
+
+            # Apply DB values if they exist
+            if db_task:
+                for model_field, placeholder in self.TASK_FIELD_MAPPING.items():
+                    value = getattr(db_task, model_field)
+                    if value:
+                        placeholder_text = "{" + placeholder + "}"
+                        task_description = task_description.replace(placeholder_text, value)
+                        expected_output = expected_output.replace(placeholder_text, value)
+
+            task_configs[task_type_lower] = ProcessedTaskConfig(
+                task_description=task_description,
+                expected_output=expected_output,
+                name=task_def.get('name') or config.get('name', task_type_lower),
+                type=task_type,
+                description=task_def.get('description') or config.get('description'),
+                verbose=self.verbose
+            )
+
+        return task_configs
+
+    def _process_tool_configs(self, specialist_config: Dict[str, Any]) -> Dict[str, ProcessedToolConfig]:
+        """Process all tool configurations"""
+        tool_configs = {}
+
+        if 'tools' not in specialist_config:
+            return tool_configs
+
+        # Get all DB tools at once
+        tool_types = [tool_def['type'] for tool_def in specialist_config['tools']]
+        db_tools = self._get_db_items(EveAITool, tool_types)
+
+        for tool_def in specialist_config['tools']:
+            tool_type = tool_def['type']
+            tool_type_lower = tool_type.lower()
+            db_tool = db_tools.get(tool_type)
+
+            # Get full configuration
+            config = cache_manager.tools_config_cache.get_config(
+                tool_type,
+                tool_def.get('version', '1.0')
+            )
+
+            # Combine configuration
+            tool_config = config.get('configuration', {})
+            if db_tool and db_tool.configuration:
+                tool_config.update(db_tool.configuration)
+
+            tool_configs[tool_type_lower] = ProcessedToolConfig(
+                name=tool_def.get('name') or config.get('name', tool_type_lower),
+                type=tool_type,
+                description=tool_def.get('description') or config.get('description'),
+                configuration=tool_config,
+                verbose=self.verbose
+            )
+
+        return tool_configs
+
+    def process_config(self) -> SpecialistProcessedConfig:
+        """Process complete specialist configuration"""
+        try:
+            # Get full specialist configuration
+            specialist_config = cache_manager.specialists_config_cache.get_config(
+                self.specialist.type,
+                self.specialist.type_version
+            )
+
+            if not specialist_config:
+                raise ValueError(f"No configuration found for {self.specialist.type}")
+
+            # Process all configurations
+            processed_config = SpecialistProcessedConfig(
+                agents=self._process_agent_configs(specialist_config),
+                tasks=self._process_task_configs(specialist_config),
+                tools=self._process_tool_configs(specialist_config)
+            )
+            return processed_config
+
+        except Exception as e:
+            current_app.logger.error(f"Error processing specialist configuration: {e}")
+            raise

common/utils/cache/crewai_configuration.py

@@ -0,0 +1,126 @@
+from dataclasses import dataclass
+from typing import Dict, Any, Optional
+
+
+@dataclass
+class ProcessedAgentConfig:
+    """Processed and ready-to-use agent configuration"""
+    role: str
+    goal: str
+    backstory: str
+    name: str
+    type: str
+    description: Optional[str] = None
+    verbose: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization"""
+        return {
+            'role': self.role,
+            'goal': self.goal,
+            'backstory': self.backstory,
+            'name': self.name,
+            'type': self.type,
+            'description': self.description,
+            'verbose': self.verbose
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedAgentConfig':
+        """Create from dictionary"""
+        return cls(**data)
+
+
+@dataclass
+class ProcessedTaskConfig:
+    """Processed and ready-to-use task configuration"""
+    task_description: str
+    expected_output: str
+    name: str
+    type: str
+    description: Optional[str] = None
+    verbose: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization"""
+        return {
+            'task_description': self.task_description,
+            'expected_output': self.expected_output,
+            'name': self.name,
+            'type': self.type,
+            'description': self.description,
+            'verbose': self.verbose
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedTaskConfig':
+        """Create from dictionary"""
+        return cls(**data)
+
+
+@dataclass
+class ProcessedToolConfig:
+    """Processed and ready-to-use tool configuration"""
+    name: str
+    type: str
+    description: Optional[str] = None
+    configuration: Optional[Dict[str, Any]] = None
+    verbose: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization"""
+        return {
+            'name': self.name,
+            'type': self.type,
+            'description': self.description,
+            'configuration': self.configuration,
+            'verbose': self.verbose
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedToolConfig':
+        """Create from dictionary"""
+        return cls(**data)
+
+
+@dataclass
+class SpecialistProcessedConfig:
+    """Complete processed configuration for a specialist"""
+    agents: Dict[str, ProcessedAgentConfig]
+    tasks: Dict[str, ProcessedTaskConfig]
+    tools: Dict[str, ProcessedToolConfig]
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert entire configuration to dictionary"""
+        return {
+            'agents': {
+                agent_type: config.to_dict()
+                for agent_type, config in self.agents.items()
+            },
+            'tasks': {
+                task_type: config.to_dict()
+                for task_type, config in self.tasks.items()
+            },
+            'tools': {
+                tool_type: config.to_dict()
+                for tool_type, config in self.tools.items()
+            }
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'SpecialistProcessedConfig':
+        """Create from dictionary"""
+        return cls(
+            agents={
+                agent_type: ProcessedAgentConfig.from_dict(config)
+                for agent_type, config in data['agents'].items()
+            },
+            tasks={
+                task_type: ProcessedTaskConfig.from_dict(config)
+                for task_type, config in data['tasks'].items()
+            },
+            tools={
+                tool_type: ProcessedToolConfig.from_dict(config)
+                for tool_type, config in data['tools'].items()
+            }
+        )

common/utils/cache/crewai_processed_config_cache.py

@@ -0,0 +1,75 @@
+from typing import Dict, Any, Type
+from flask import current_app
+
+from common.utils.cache.base import CacheHandler
+from common.utils.cache.crewai_configuration import SpecialistProcessedConfig
+from common.utils.cache.crewai_config_processor import BaseCrewAIConfigProcessor
+
+
+class CrewAIProcessedConfigCacheHandler(CacheHandler[SpecialistProcessedConfig]):
+    """Handles caching of processed specialist configurations"""
+    handler_name = 'crewai_processed_config_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'crewai_processed_config')
+        self.configure_keys('tenant_id', 'specialist_id')
+
+    def _to_cache_data(self, instance: SpecialistProcessedConfig) -> Dict[str, Any]:
+        """Convert SpecialistProcessedConfig to cache data"""
+        return instance.to_dict()
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> SpecialistProcessedConfig:
+        """Create SpecialistProcessedConfig from cache data"""
+        return SpecialistProcessedConfig.from_dict(data)
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """Validate cache data"""
+        required_keys = {'agents', 'tasks', 'tools'}
+        if not all(key in value for key in required_keys):
+            current_app.logger.warning(f'CrewAI Processed Config Cache missing required keys: {required_keys}')
+            return False
+        return bool(value['agents'] or value['tasks'])
+
+    def get_specialist_config(self, tenant_id: int, specialist_id: int) -> SpecialistProcessedConfig:
+        """
+        Get or create processed configuration for a specialist
+
+        Args:
+            tenant_id: Tenant ID
+            specialist_id: Specialist ID
+
+        Returns:
+            Processed specialist configuration
+
+        Raises:
+            ValueError: If specialist not found or processor not configured
+        """
+
+        def creator_func(tenant_id: int, specialist_id: int) -> SpecialistProcessedConfig:
+            # Create processor instance and process config
+            processor = BaseCrewAIConfigProcessor(tenant_id, specialist_id)
+            return processor.process_config()
+
+        return self.get(
+            creator_func,
+            tenant_id=tenant_id,
+            specialist_id=specialist_id
+        )
+
+    def invalidate_tenant_specialist(self, tenant_id: int, specialist_id: int):
+        """Invalidate cache for a specific tenant's specialist"""
+        self.invalidate(
+            tenant_id=tenant_id,
+            specialist_id=specialist_id
+        )
+        current_app.logger.info(
+            f"Invalidated cache for tenant {tenant_id} specialist {specialist_id}"
+        )
+
+
+def register_specialist_cache_handlers(cache_manager) -> None:
+    """Register specialist cache handlers with cache manager"""
+    cache_manager.register_handler(
+        CrewAIProcessedConfigCacheHandler,
+        'eveai_chat_workers'
+    )

common/utils/cache/eveai_cache_manager.py

@@ -0,0 +1,51 @@
+from typing import Type
+
+from flask import Flask
+
+from common.utils.cache.base import CacheHandler
+from common.utils.cache.regions import create_cache_regions
+from common.utils.cache.config_cache import AgentConfigCacheHandler
+
+
+class EveAICacheManager:
+    """Cache manager with registration capabilities"""
+
+    def __init__(self):
+        self._regions = {}
+        self._handlers = {}
+        self._handler_instances = {}
+
+    def init_app(self, app: Flask):
+        """Initialize cache regions"""
+        self._regions = create_cache_regions(app)
+
+        # Store regions in instance
+        for region_name, region in self._regions.items():
+            setattr(self, f"{region_name}_region", region)
+
+        app.logger.info(f'Cache regions initialized: {self._regions.keys()}')
+
+    def register_handler(self, handler_class: Type[CacheHandler], region: str):
+        """Register a cache handler class with its region"""
+        if not hasattr(handler_class, 'handler_name'):
+            raise ValueError("Cache handler must define handler_name class attribute")
+        self._handlers[handler_class] = region
+
+        # Create handler instance
+        region_instance = self._regions[region]
+        handler_instance = handler_class(region_instance)
+        self._handler_instances[handler_class.handler_name] = handler_instance
+
+    def invalidate_region(self, region_name: str):
+        """Invalidate an entire cache region"""
+        if region_name in self._regions:
+            self._regions[region_name].invalidate()
+        else:
+            raise ValueError(f"Unknown cache region: {region_name}")
+
+    def __getattr__(self, name):
+        """Handle dynamic access to registered handlers"""
+        instances = object.__getattribute__(self, '_handler_instances')
+        if name in instances:
+            return instances[name]
+        raise AttributeError(f"'EveAICacheManager' object has no attribute '{name}'")

common/utils/cache/license_cache.py

@@ -0,0 +1,102 @@
+# common/utils/cache/license_cache.py
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+
+from flask import current_app
+from sqlalchemy import and_
+from sqlalchemy.inspection import inspect
+
+from common.utils.cache.base import CacheHandler
+from common.models.entitlements import License
+
+
+class LicenseCacheHandler(CacheHandler[License]):
+    """Handles caching of active licenses for tenants"""
+    handler_name = 'license_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'active_license')
+        self.configure_keys('tenant_id')
+
+    def _to_cache_data(self, instance: License) -> Dict[str, Any]:
+        """Convert License instance to cache data using SQLAlchemy inspection"""
+        if not instance:
+            return {}
+
+        # Get all column attributes from the SQLAlchemy model
+        mapper = inspect(License)
+        data = {}
+
+        for column in mapper.columns:
+            value = getattr(instance, column.name)
+
+            # Handle date serialization
+            if isinstance(value, dt):
+                data[column.name] = value.isoformat()
+            else:
+                data[column.name] = value
+
+        return data
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> License:
+        """Create License instance from cache data using SQLAlchemy inspection"""
+        if not data:
+            return None
+
+        # Create a new License instance
+        license = License()
+        mapper = inspect(License)
+
+        # Set all attributes dynamically
+        for column in mapper.columns:
+            if column.name in data:
+                value = data[column.name]
+
+                # Handle date deserialization
+                if column.name.endswith('_date') and value:
+                    if isinstance(value, str):
+                        value = dt.fromisoformat(value).date()
+
+                setattr(license, column.name, value)
+
+        return license
+
+    def _should_cache(self, value: License) -> bool:
+        """Validate if the license should be cached"""
+        return value is not None and value.id is not None
+
+    def get_active_license(self, tenant_id: int) -> Optional[License]:
+        """
+        Get the currently active license for a tenant
+
+        Args:
+            tenant_id: ID of the tenant
+
+        Returns:
+            License instance if found, None otherwise
+        """
+
+        def creator_func(tenant_id: int) -> Optional[License]:
+            from common.extensions import db
+            current_date = dt.now(tz=tz.utc).date()
+
+            # TODO --> Active License via active Period?
+
+            return (db.session.query(License)
+                    .filter_by(tenant_id=tenant_id)
+                    .filter(License.start_date <= current_date)
+                    .last())
+
+        return self.get(creator_func, tenant_id=tenant_id)
+
+    def invalidate_tenant_license(self, tenant_id: int):
+        """Invalidate cached license for specific tenant"""
+        self.invalidate(tenant_id=tenant_id)
+
+
+def register_license_cache_handlers(cache_manager) -> None:
+    """Register license cache handlers with cache manager"""
+    cache_manager.register_handler(
+        LicenseCacheHandler,
+        'eveai_model'  # Use existing eveai_model region
+    )

common/utils/cache/regions.py

@@ -0,0 +1,74 @@
+# common/utils/cache/regions.py
+import time
+
+from dogpile.cache import make_region
+from urllib.parse import urlparse
+import os
+
+
+def get_redis_config(app):
+    """
+    Create Redis configuration dict based on app config
+    Handles both authenticated and non-authenticated setups
+    """
+    # Parse the REDIS_BASE_URI to get all components
+    redis_uri = urlparse(app.config['REDIS_BASE_URI'])
+
+    config = {
+        'host': redis_uri.hostname,
+        'port': int(redis_uri.port or 6379),
+        'db': 4,  # Keep this for later use
+        'redis_expiration_time': 3600,
+        'distributed_lock': True,
+        'thread_local_lock': False,
+    }
+
+    # Add authentication if provided
+    if redis_uri.username and redis_uri.password:
+        config.update({
+            'username': redis_uri.username,
+            'password': redis_uri.password
+        })
+
+    return config
+
+
+def create_cache_regions(app):
+    """Initialize all cache regions with app config"""
+    redis_config = get_redis_config(app)
+    regions = {}
+    startup_time = int(time.time())
+
+    # Region for model-related caching (ModelVariables etc)
+    model_region = make_region(name='eveai_model').configure(
+        'dogpile.cache.redis',
+        arguments=redis_config,
+        replace_existing_backend=True
+    )
+    regions['eveai_model'] = model_region
+
+    # Region for eveai_chat_workers components (Specialists, Retrievers, ...)
+    eveai_chat_workers_region = make_region(name='eveai_chat_workers').configure(
+        'dogpile.cache.redis',
+        arguments=redis_config,  # arguments={**redis_config, 'db': 4},  # Different DB
+        replace_existing_backend=True
+    )
+    regions['eveai_chat_workers'] = eveai_chat_workers_region
+
+    # Region for eveai_workers components (Processors, ...)
+    eveai_workers_region = make_region(name='eveai_workers').configure(
+        'dogpile.cache.redis',
+        arguments=redis_config,  # Same config for now
+        replace_existing_backend=True
+    )
+    regions['eveai_workers'] = eveai_workers_region
+
+    eveai_config_region = make_region(name='eveai_config').configure(
+        'dogpile.cache.redis',
+        arguments=redis_config,
+        replace_existing_backend=True
+    )
+    regions['eveai_config'] = eveai_config_region
+
+    return regions
+

common/utils/celery_utils.py

@@ -1,14 +1,14 @@
 from celery import Celery
 from kombu import Queue
 from werkzeug.local import LocalProxy
+from redbeat import RedBeatScheduler
 
 celery_app = Celery()
 
 
-def init_celery(celery, app):
+def init_celery(celery, app, is_beat=False):
     celery_app.main = app.name
-    app.logger.debug(f'CELERY_BROKER_URL: {app.config["CELERY_BROKER_URL"]}')
-    app.logger.debug(f'CELERY_RESULT_BACKEND: {app.config["CELERY_RESULT_BACKEND"]}')
+
     celery_config = {
         'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),
         'result_backend': app.config.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0'),
@@ -17,19 +17,40 @@ def init_celery(celery, app):
         'accept_content': app.config.get('CELERY_ACCEPT_CONTENT', ['json']),
         'timezone': app.config.get('CELERY_TIMEZONE', 'UTC'),
         'enable_utc': app.config.get('CELERY_ENABLE_UTC', True),
-        'task_routes': {'eveai_worker.tasks.create_embeddings': {'queue': 'embeddings',
-                                                                 'routing_key': 'embeddings.create_embeddings'}},
     }
+
+    if is_beat:
+        # Add configurations specific to Beat scheduler
+        celery_config['beat_scheduler'] = 'redbeat.RedBeatScheduler'
+        celery_config['redbeat_lock_key'] = 'redbeat::lock'
+        celery_config['beat_max_loop_interval'] = 10  # Adjust as needed
+
     celery_app.conf.update(**celery_config)
 
-    # Setting up Celery task queues
-    celery_app.conf.task_queues = (
-        Queue('default', routing_key='task.#'),
-        Queue('embeddings', routing_key='embeddings.#', queue_arguments={'x-max-priority': 10}),
-        Queue('llm_interactions', routing_key='llm_interactions.#', queue_arguments={'x-max-priority': 5}),
-    )
+    # Task queues for workers only
+    if not is_beat:
+        celery_app.conf.task_queues = (
+            Queue('default', routing_key='task.#'),
+            Queue('embeddings', routing_key='embeddings.#', queue_arguments={'x-max-priority': 10}),
+            Queue('llm_interactions', routing_key='llm_interactions.#', queue_arguments={'x-max-priority': 5}),
+            Queue('entitlements', routing_key='entitlements.#', queue_arguments={'x-max-priority': 10}),
+        )
+        celery_app.conf.task_routes = {
+            'eveai_workers.*': {  # All tasks from eveai_workers module
+                'queue': 'embeddings',
+                'routing_key': 'embeddings.#',
+            },
+            'eveai_chat_workers.*': {  # All tasks from eveai_chat_workers module
+                'queue': 'llm_interactions',
+                'routing_key': 'llm_interactions.#',
+            },
+            'eveai_entitlements.*': {  # All tasks from eveai_entitlements module
+                'queue': 'entitlements',
+                'routing_key': 'entitlements.#',
+            }
+        }
 
-    # Ensuring tasks execute with Flask application context
+    # Ensure tasks execute with Flask context
     class ContextTask(celery.Task):
         def __call__(self, *args, **kwargs):
             with app.app_context():

common/utils/chat_utils.py

@@ -0,0 +1,42 @@
+"""
+Utility functions for chat customization.
+"""
+
+def get_default_chat_customisation(tenant_customisation=None):
+    """
+    Get chat customization options with default values for missing options.
+    
+    Args:
+        tenant_customization (dict, optional): The tenant's customization options.
+            Defaults to None.
+            
+    Returns:
+        dict: A dictionary containing all customization options with default values
+            for any missing options.
+    """
+    # Default customization options
+    default_customisation = {
+        'primary_color': '#007bff',
+        'secondary_color': '#6c757d',
+        'background_color': '#ffffff',
+        'text_color': '#212529',
+        'sidebar_color': '#f8f9fa',
+        'logo_url': None,
+        'sidebar_text': None,
+        'welcome_message': 'Hello! How can I help you today?',
+        'team_info': []
+    }
+    
+    # If no tenant customization is provided, return the defaults
+    if tenant_customisation is None:
+        return default_customisation
+    
+    # Start with the default customization
+    customisation = default_customisation.copy()
+    
+    # Update with tenant customization
+    for key, value in tenant_customisation.items():
+        if key in customisation:
+            customisation[key] = value
+    
+    return customisation

common/utils/config_field_types.py

@@ -0,0 +1,711 @@
+from typing import Optional, List, Union, Dict, Any, Pattern
+from pydantic import BaseModel, field_validator, model_validator
+from typing_extensions import Annotated
+import re
+from datetime import datetime
+import json
+from textwrap import dedent
+import yaml
+from dataclasses import dataclass
+
+
+class TaggingField(BaseModel):
+    """Represents a single tagging field configuration"""
+    type: str
+    required: bool = False
+    description: Optional[str] = None
+    allowed_values: Optional[List[Any]] = None  # for enum type
+    min_value: Optional[Union[int, float]] = None  # for numeric types
+    max_value: Optional[Union[int, float]] = None  # for numeric types
+
+    @field_validator('type', mode='before')
+    @classmethod
+    def validate_type(cls, v: str) -> str:
+        valid_types = ['string', 'integer', 'float', 'date', 'enum', 'color']
+        if v not in valid_types:
+            raise ValueError(f'type must be one of {valid_types}')
+        return v
+
+    @model_validator(mode='after')
+    def validate_field_constraints(self) -> 'TaggingField':
+        # Validate enum constraints
+        if self.type == 'enum':
+            if not self.allowed_values:
+                raise ValueError('allowed_values must be provided for enum type')
+        elif self.allowed_values is not None:
+            raise ValueError('allowed_values only valid for enum type')
+
+        # Validate numeric constraints
+        if self.type not in ('integer', 'float'):
+            if self.min_value is not None or self.max_value is not None:
+                raise ValueError('min_value/max_value only valid for numeric types')
+        else:
+            if self.min_value is not None and self.max_value is not None and self.min_value >= self.max_value:
+                raise ValueError('min_value must be less than max_value')
+
+        return self
+
+
+class TaggingFields(BaseModel):
+    """Represents a collection of tagging fields, mapped by their names"""
+    fields: Dict[str, TaggingField]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Dict[str, Any]]) -> 'TaggingFields':
+        return cls(fields={
+            field_name: TaggingField(**field_config)
+            for field_name, field_config in data.items()
+        })
+
+    def to_dict(self) -> Dict[str, Dict[str, Any]]:
+        return {
+            field_name: field.model_dump(exclude_none=True)
+            for field_name, field in self.fields.items()
+        }
+
+
+class ChunkingPatternsField(BaseModel):
+    """Represents a set of chunking patterns"""
+    patterns: List[str]
+
+    @field_validator('patterns')
+    def validate_patterns(cls, patterns):
+        for pattern in patterns:
+            try:
+                re.compile(pattern)
+            except re.error as e:
+                raise ValueError(f"Invalid regex pattern '{pattern}': {str(e)}")
+        return patterns
+
+
+class ArgumentConstraint(BaseModel):
+    """Base class for all argument constraints"""
+    description: Optional[str] = None
+    error_message: Optional[str] = None
+
+
+class NumericConstraint(ArgumentConstraint):
+    """Constraints for numeric values (int/float)"""
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+    include_min: bool = True  # True for >= min_value, False for > min_value
+    include_max: bool = True  # True for <= max_value, False for < max_value
+
+    @model_validator(mode='after')
+    def validate_ranges(self) -> 'NumericConstraint':
+        if self.min_value is not None and self.max_value is not None:
+            if self.min_value > self.max_value:
+                raise ValueError("min_value must be less than or equal to max_value")
+        return self
+
+    def validate(self, value: Union[int, float]) -> bool:
+        if self.min_value is not None:
+            if self.include_min and value < self.min_value:
+                return False
+            if not self.include_min and value <= self.min_value:
+                return False
+        if self.max_value is not None:
+            if self.include_max and value > self.max_value:
+                return False
+            if not self.include_max and value >= self.max_value:
+                return False
+        return True
+
+
+class StringConstraint(ArgumentConstraint):
+    """Constraints for string values"""
+    min_length: Optional[int] = None
+    max_length: Optional[int] = None
+    patterns: Optional[List[str]] = None  # List of regex patterns to match
+    pattern_match_all: bool = False  # If True, string must match all patterns
+    forbidden_patterns: Optional[List[str]] = None  # List of regex patterns that must not match
+    allow_empty: bool = False
+
+    @field_validator('patterns', 'forbidden_patterns')
+    @classmethod
+    def validate_patterns(cls, v: Optional[List[str]]) -> Optional[List[str]]:
+        if v is not None:
+            # Validate each pattern compiles
+            for pattern in v:
+                try:
+                    re.compile(pattern)
+                except re.error as e:
+                    raise ValueError(f"Invalid regex pattern '{pattern}': {str(e)}")
+        return v
+
+    def validate(self, value: str) -> bool:
+        if not self.allow_empty and not value:
+            return False
+
+        if self.min_length is not None and len(value) < self.min_length:
+            return False
+
+        if self.max_length is not None and len(value) > self.max_length:
+            return False
+
+        if self.patterns:
+            matches = [bool(re.search(pattern, value)) for pattern in self.patterns]
+            if self.pattern_match_all and not all(matches):
+                return False
+            if not self.pattern_match_all and not any(matches):
+                return False
+
+        if self.forbidden_patterns:
+            for pattern in self.forbidden_patterns:
+                if re.search(pattern, value):
+                    return False
+
+        return True
+
+
+class DateConstraint(ArgumentConstraint):
+    """Constraints for date values"""
+    min_date: Optional[datetime] = None
+    max_date: Optional[datetime] = None
+    include_min: bool = True
+    include_max: bool = True
+    allowed_formats: Optional[List[str]] = None  # List of allowed date formats
+
+    @model_validator(mode='after')
+    def validate_ranges(self) -> 'DateConstraint':
+        if self.min_date and self.max_date and self.min_date > self.max_date:
+            raise ValueError("min_date must be less than or equal to max_date")
+        return self
+
+    def validate(self, value: datetime) -> bool:
+        if self.min_date is not None:
+            if self.include_min and value < self.min_date:
+                return False
+            if not self.include_min and value <= self.min_date:
+                return False
+
+        if self.max_date is not None:
+            if self.include_max and value > self.max_date:
+                return False
+            if not self.include_max and value >= self.max_date:
+                return False
+
+        return True
+
+
+class EnumConstraint(ArgumentConstraint):
+    """Constraints for enum values"""
+    allowed_values: List[Any]
+    case_sensitive: bool = True  # For string enums
+    allow_multiple: bool = False  # If True, value can be a list of allowed values
+    min_selections: Optional[int] = None  # When allow_multiple is True
+    max_selections: Optional[int] = None  # When allow_multiple is True
+
+    @model_validator(mode='after')
+    def validate_selections(self) -> 'EnumConstraint':
+        if self.allow_multiple:
+            if self.min_selections is not None and self.max_selections is not None:
+                if self.min_selections > self.max_selections:
+                    raise ValueError("min_selections must be less than or equal to max_selections")
+                if self.max_selections > len(self.allowed_values):
+                    raise ValueError("max_selections cannot be greater than number of allowed values")
+        return self
+
+    def validate(self, value: Union[Any, List[Any]]) -> bool:
+        if self.allow_multiple:
+            if not isinstance(value, list):
+                return False
+
+            if self.min_selections is not None and len(value) < self.min_selections:
+                return False
+
+            if self.max_selections is not None and len(value) > self.max_selections:
+                return False
+
+            for v in value:
+                if not self._validate_single_value(v):
+                    return False
+        else:
+            return self._validate_single_value(value)
+
+        return True
+
+    def _validate_single_value(self, value: Any) -> bool:
+        if isinstance(value, str) and not self.case_sensitive:
+            return any(str(value).lower() == str(v).lower() for v in self.allowed_values)
+        return value in self.allowed_values
+
+
+class ArgumentDefinition(BaseModel):
+    """Defines an argument with its type and constraints"""
+    name: str
+    type: str
+    description: Optional[str] = None
+    required: bool = False
+    default: Optional[Any] = None
+    constraints: Optional[Union[NumericConstraint, StringConstraint, DateConstraint, EnumConstraint]] = None
+
+    @field_validator('type')
+    @classmethod
+    def validate_type(cls, v: str) -> str:
+        valid_types = ['string', 'integer', 'float', 'date', 'enum', 'color']
+        if v not in valid_types:
+            raise ValueError(f'type must be one of {valid_types}')
+        return v
+
+    @model_validator(mode='after')
+    def validate_constraints(self) -> 'ArgumentDefinition':
+        if self.constraints:
+            expected_constraint_types = {
+                'string': StringConstraint,
+                'integer': NumericConstraint,
+                'float': NumericConstraint,
+                'date': DateConstraint,
+                'enum': EnumConstraint,
+                'color': StringConstraint
+            }
+
+            expected_type = expected_constraint_types.get(self.type)
+            if not isinstance(self.constraints, expected_type):
+                raise ValueError(f'Constraints for type {self.type} must be of type {expected_type.__name__}')
+
+            if self.default is not None:
+                if not self.constraints.validate(self.default):
+                    raise ValueError(f'Default value does not satisfy constraints for {self.name}')
+
+        return self
+
+
+class ArgumentDefinitions(BaseModel):
+    """Collection of argument definitions"""
+    arguments: Dict[str, ArgumentDefinition]
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Dict[str, Any]]) -> 'ArgumentDefinitions':
+        return cls(arguments={
+            arg_name: ArgumentDefinition(**arg_config)
+            for arg_name, arg_config in data.items()
+        })
+
+    def to_dict(self) -> Dict[str, Dict[str, Any]]:
+        return {
+            arg_name: arg.model_dump(exclude_none=True)
+            for arg_name, arg in self.arguments.items()
+        }
+
+    def validate_argument_values(self, values: Dict[str, Any]) -> Dict[str, str]:
+        """
+        Validate a set of argument values against their definitions
+        Returns a dictionary of error messages for invalid arguments
+        """
+        errors = {}
+
+        # Check for required arguments
+        for name, arg_def in self.arguments.items():
+            if arg_def.required and name not in values:
+                errors[name] = "Required argument missing"
+                continue
+
+            if name in values:
+                value = values[name]
+
+                # Validate type
+                try:
+                    if arg_def.type == 'integer':
+                        value = int(value)
+                    elif arg_def.type == 'float':
+                        value = float(value)
+                    elif arg_def.type == 'date' and isinstance(value, str):
+                        if arg_def.constraints and arg_def.constraints.allowed_formats:
+                            for fmt in arg_def.constraints.allowed_formats:
+                                try:
+                                    value = datetime.strptime(value, fmt)
+                                    break
+                                except ValueError:
+                                    continue
+                            else:
+                                errors[
+                                    name] = f"Invalid date format. Allowed formats: {arg_def.constraints.allowed_formats}"
+                                continue
+                except (ValueError, TypeError):
+                    errors[name] = f"Invalid type. Expected {arg_def.type}"
+                    continue
+
+                # Validate constraints
+                if arg_def.constraints and not arg_def.constraints.validate(value):
+                    errors[name] = arg_def.constraints.error_message or "Value does not satisfy constraints"
+
+        return errors
+
+
+@dataclass
+class DocumentationFormat:
+    """Constants for documentation formats"""
+    MARKDOWN = "markdown"
+    JSON = "json"
+    YAML = "yaml"
+
+
+@dataclass
+class DocumentationVersion:
+    """Constants for documentation versions"""
+    BASIC = "basic"  # Original documentation without retriever info
+    EXTENDED = "extended"  # Including retriever documentation
+
+
+def _generate_argument_constraints(field_config: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Generate possible argument constraints based on field type"""
+    constraints = []
+
+    base_constraint = {
+        "description": f"Constraint for {field_config.get('description', 'field')}",
+        "error_message": "Optional custom error message"
+    }
+
+    if field_config["type"] == "integer" or field_config["type"] == "float":
+        constraints.append({
+            **base_constraint,
+            "type": "NumericConstraint",
+            "possible_constraints": {
+                "min_value": "number",
+                "max_value": "number",
+                "include_min": "boolean",
+                "include_max": "boolean"
+            },
+            "example": {
+                "min_value": field_config.get("min_value", 0),
+                "max_value": field_config.get("max_value", 100),
+                "include_min": True,
+                "include_max": True
+            }
+        })
+
+    elif field_config["type"] == "string":
+        constraints.append({
+            **base_constraint,
+            "type": "StringConstraint",
+            "possible_constraints": {
+                "min_length": "integer",
+                "max_length": "integer",
+                "patterns": "list[str]",
+                "pattern_match_all": "boolean",
+                "forbidden_patterns": "list[str]",
+                "allow_empty": "boolean"
+            },
+            "example": {
+                "min_length": 1,
+                "max_length": 100,
+                "patterns": ["^[A-Za-z0-9]+$"],
+                "pattern_match_all": False,
+                "forbidden_patterns": ["^test_", "_temp$"],
+                "allow_empty": False
+            }
+        })
+
+    elif field_config["type"] == "enum":
+        constraints.append({
+            **base_constraint,
+            "type": "EnumConstraint",
+            "possible_constraints": {
+                "allowed_values": f"list[{field_config.get('allowed_values', ['value1', 'value2'])}]",
+                "case_sensitive": "boolean",
+                "allow_multiple": "boolean",
+                "min_selections": "integer",
+                "max_selections": "integer"
+            },
+            "example": {
+                "allowed_values": field_config.get("allowed_values", ["value1", "value2"]),
+                "case_sensitive": True,
+                "allow_multiple": True,
+                "min_selections": 1,
+                "max_selections": 2
+            }
+        })
+
+    elif field_config["type"] == "date":
+        constraints.append({
+            **base_constraint,
+            "type": "DateConstraint",
+            "possible_constraints": {
+                "min_date": "datetime",
+                "max_date": "datetime",
+                "include_min": "boolean",
+                "include_max": "boolean",
+                "allowed_formats": "list[str]"
+            },
+            "example": {
+                "min_date": "2024-01-01T00:00:00",
+                "max_date": "2024-12-31T23:59:59",
+                "include_min": True,
+                "include_max": True,
+                "allowed_formats": ["%Y-%m-%d", "%Y/%m/%d"]
+            }
+        })
+
+    return constraints
+
+
+def generate_field_documentation(
+        tagging_fields: Dict[str, Any],
+        format: str = "markdown",
+        version: str = "basic"
+) -> str:
+    """
+    Generate documentation for tagging fields configuration.
+
+    Args:
+        tagging_fields: Dictionary containing tagging fields configuration
+        format: Output format ("markdown", "json", or "yaml")
+        version: Documentation version ("basic" or "extended")
+
+    Returns:
+        str: Formatted documentation
+    """
+    if version not in [DocumentationVersion.BASIC, DocumentationVersion.EXTENDED]:
+        raise ValueError(f"Unsupported documentation version: {version}")
+
+    # Normalize fields configuration
+    normalized_fields = {}
+
+    for field_name, field_config in tagging_fields.items():
+        field_doc = {
+            "name": field_name,
+            "type": field_config["type"],
+            "required": field_config.get("required", False),
+            "description": field_config.get("description", "No description provided"),
+            "constraints": []
+        }
+
+        # Only include possible arguments in extended version
+        if version == DocumentationVersion.EXTENDED:
+            field_doc["possible_arguments"] = _generate_argument_constraints(field_config)
+
+        # Add type-specific constraints
+        if field_config["type"] == "integer" or field_config["type"] == "float":
+            if "min_value" in field_config:
+                field_doc["constraints"].append(
+                    f"Minimum value: {field_config['min_value']}")
+            if "max_value" in field_config:
+                field_doc["constraints"].append(
+                    f"Maximum value: {field_config['max_value']}")
+
+        elif field_config["type"] == "string":
+            if "min_length" in field_config:
+                field_doc["constraints"].append(
+                    f"Minimum length: {field_config['min_length']}")
+            if "max_length" in field_config:
+                field_doc["constraints"].append(
+                    f"Maximum length: {field_config['max_length']}")
+            if "patterns" in field_config:
+                field_doc["constraints"].append(
+                    f"Must match patterns: {', '.join(field_config['patterns'])}")
+
+        elif field_config["type"] == "enum":
+            if "allowed_values" in field_config:
+                field_doc["constraints"].append(
+                    f"Allowed values: {', '.join(str(v) for v in field_config['allowed_values'])}")
+
+        elif field_config["type"] == "date":
+            if "min_date" in field_config:
+                field_doc["constraints"].append(
+                    f"Minimum date: {field_config['min_date']}")
+            if "max_date" in field_config:
+                field_doc["constraints"].append(
+                    f"Maximum date: {field_config['max_date']}")
+            if "allowed_formats" in field_config:
+                field_doc["constraints"].append(
+                    f"Allowed formats: {', '.join(field_config['allowed_formats'])}")
+
+        normalized_fields[field_name] = field_doc
+
+    # Generate documentation in requested format
+    if format == DocumentationFormat.MARKDOWN:
+        return _generate_markdown_docs(normalized_fields, version)
+    elif format == DocumentationFormat.JSON:
+        return _generate_json_docs(normalized_fields, version)
+    elif format == DocumentationFormat.YAML:
+        return _generate_yaml_docs(normalized_fields, version)
+    else:
+        raise ValueError(f"Unsupported documentation format: {format}")
+
+
+def _generate_markdown_docs(fields: Dict[str, Any], version: str) -> str:
+    """Generate markdown documentation"""
+    docs = ["# Tagging Fields Documentation\n"]
+
+    # Add overview table
+    docs.append("## Fields Overview\n")
+    docs.append("| Field Name | Type | Required | Description |")
+    docs.append("|------------|------|----------|-------------|")
+
+    for field_name, field in fields.items():
+        docs.append(
+            f"| {field_name} | {field['type']} | "
+            f"{'Yes' if field['required'] else 'No'} | {field['description']} |"
+        )
+
+    # Add detailed field specifications
+    docs.append("\n## Detailed Field Specifications\n")
+
+    for field_name, field in fields.items():
+        docs.append(f"### {field_name}\n")
+        docs.append(f"**Type:** {field['type']}")
+        docs.append(f"**Required:** {'Yes' if field['required'] else 'No'}")
+        docs.append(f"**Description:** {field['description']}\n")
+
+        if field["constraints"]:
+            docs.append("**Field Constraints:**")
+            for constraint in field["constraints"]:
+                docs.append(f"- {constraint}")
+            docs.append("")
+
+        # Add retriever argument documentation only in extended version
+        if version == DocumentationVersion.EXTENDED and "possible_arguments" in field:
+            docs.append("**Possible Retriever Arguments:**")
+            for arg_constraint in field["possible_arguments"]:
+                docs.append(f"\n*{arg_constraint['type']}*")
+                docs.append(f"Description: {arg_constraint['description']}")
+                docs.append("\nPossible constraints:")
+                for const_name, const_type in arg_constraint["possible_constraints"].items():
+                    docs.append(f"- `{const_name}`: {const_type}")
+
+                docs.append("\nExample:")
+                docs.append("```python")
+                docs.append(json.dumps(arg_constraint["example"], indent=2))
+                docs.append("```\n")
+
+    # Add example retriever configuration only in extended version
+    if version == DocumentationVersion.EXTENDED:
+        docs.append("\n## Example Retriever Configuration\n")
+        docs.append("```python")
+        example_config = {
+            "metadata_filters": {
+                field_name: field["possible_arguments"][0]["example"]
+                for field_name, field in fields.items()
+                if "possible_arguments" in field
+            }
+        }
+        docs.append(json.dumps(example_config, indent=2))
+        docs.append("```")
+
+    return "\n".join(docs)
+
+
+def _generate_json_docs(fields: Dict[str, Any], version: str) -> str:
+    """Generate JSON documentation"""
+    doc = {
+        "tagging_fields_documentation": {
+            "version": version,
+            "fields": fields
+        }
+    }
+
+    if version == DocumentationVersion.EXTENDED:
+        doc["tagging_fields_documentation"]["example_retriever_config"] = {
+            "metadata_filters": {
+                field_name: field["possible_arguments"][0]["example"]
+                for field_name, field in fields.items()
+                if "possible_arguments" in field
+            }
+        }
+
+    return json.dumps(doc, indent=2)
+
+
+def _generate_yaml_docs(fields: Dict[str, Any], version: str) -> str:
+    """Generate YAML documentation"""
+    doc = {
+        "tagging_fields_documentation": {
+            "version": version,
+            "fields": fields
+        }
+    }
+
+    if version == DocumentationVersion.EXTENDED:
+        doc["tagging_fields_documentation"]["example_retriever_config"] = {
+            "metadata_filters": {
+                field_name: field["possible_arguments"][0]["example"]
+                for field_name, field in fields.items()
+                if "possible_arguments" in field
+            }
+        }
+
+    return yaml.dump(doc, sort_keys=False, default_flow_style=False)
+
+
+def patterns_to_json(text_area_content: str) -> str:
+    """Convert line-based patterns to JSON"""
+    text_area_content = text_area_content.strip()
+    if len(text_area_content) == 0:
+        return json.dumps([])
+    # Split on newlines and remove empty lines
+    patterns = [line.strip() for line in text_area_content.split('\n') if line.strip()]
+    return json.dumps(patterns)
+
+
+def json_to_patterns(json_content: str) -> str:
+    """Convert JSON patterns list to text area content"""
+    try:
+        patterns = json.loads(json_content)
+        if not isinstance(patterns, list):
+            raise ValueError("JSON must contain a list of patterns")
+        # Join with newlines
+        return '\n'.join(patterns)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON format: {e}")
+
+
+def json_to_pattern_list(json_content: str) -> list:
+    """Convert JSON patterns list to text area content"""
+    try:
+        if json_content:
+            patterns = json.loads(json_content)
+            if not isinstance(patterns, list):
+                raise ValueError("JSON must contain a list of patterns")
+            # Unescape if needed
+            patterns = [pattern.replace('\\\\', '\\') for pattern in patterns]
+            return patterns
+        else:
+            return []
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON format: {e}")
+
+
+def normalize_json_field(value: str | dict | None, field_name: str = "JSON field") -> dict:
+    """
+    Normalize a JSON field value to ensure it's a valid dictionary.
+
+    Args:
+        value: The input value which can be:
+            - None (will return empty dict)
+            - String (will be parsed as JSON)
+            - Dict (will be validated and returned)
+        field_name: Name of the field for error messages
+
+    Returns:
+        dict: The normalized JSON data as a Python dictionary
+
+    Raises:
+        ValueError: If the input string is not valid JSON or the input dict contains invalid types
+    """
+    # Handle None case
+    if value is None:
+        return {}
+
+    # Handle dictionary case
+    if isinstance(value, dict):
+        try:
+            # Validate all values are JSON serializable
+            import json
+            json.dumps(value)
+            return value
+        except TypeError as e:
+            raise ValueError(f"{field_name} contains invalid types: {str(e)}")
+
+    # Handle string case
+    if isinstance(value, str):
+        if not value.strip():
+            return {}
+
+        try:
+            import json
+            return json.loads(value)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"{field_name} contains invalid JSON: {str(e)}")
+
+    raise ValueError(f"{field_name} must be a string, dictionary, or None (got {type(value)})")

common/utils/content_utils.py

@@ -0,0 +1,215 @@
+import os
+import re
+import logging
+from packaging import version
+from flask import current_app
+
+logger = logging.getLogger(__name__)
+
+class ContentManager:
+    def __init__(self, app=None):
+        self.app = app
+        if app:
+            self.init_app(app)
+
+    def init_app(self, app):
+        self.app = app
+
+        # Controleer of het pad bestaat
+        if not os.path.exists(app.config['CONTENT_DIR']):
+            logger.warning(f"Content directory not found at: {app.config['CONTENT_DIR']}")
+        else:
+            logger.info(f"Content directory configured at: {app.config['CONTENT_DIR']}")
+
+    def get_content_path(self, content_type, major_minor=None, patch=None):
+        """
+        Geef het volledige pad naar een contentbestand
+
+        Args:
+            content_type (str): Type content (bv. 'changelog', 'terms')
+            major_minor (str, optional): Major.Minor versie (bv. '1.0')
+            patch (str, optional): Patchnummer (bv. '5')
+
+        Returns:
+            str: Volledige pad naar de content map of bestand
+        """
+        content_path = os.path.join(self.app.config['CONTENT_DIR'], content_type)
+
+        if major_minor:
+            content_path = os.path.join(content_path, major_minor)
+
+            if patch:
+                content_path = os.path.join(content_path, f"{major_minor}.{patch}.md")
+
+        return content_path
+
+    def _parse_version(self, filename):
+        """Parse een versienummer uit een bestandsnaam"""
+        match = re.match(r'(\d+\.\d+)\.(\d+)\.md', filename)
+        if match:
+            return match.group(1), match.group(2)
+        return None, None
+
+    def get_latest_version(self, content_type, major_minor=None):
+        """
+        Verkrijg de laatste versie van een bepaald contenttype
+
+        Args:
+            content_type (str): Type content (bv. 'changelog', 'terms')
+            major_minor (str, optional): Specifieke major.minor versie, anders de hoogste
+
+        Returns:
+            tuple: (major_minor, patch, full_version) of None als niet gevonden
+        """
+        try:
+            # Basispad voor dit contenttype
+            content_path = os.path.join(self.app.config['CONTENT_DIR'], content_type)
+
+            if not os.path.exists(content_path):
+                logger.error(f"Content path does not exist: {content_path}")
+                return None
+
+            # Als geen major_minor opgegeven, vind de hoogste
+            if not major_minor:
+                available_versions = os.listdir(content_path)
+                if not available_versions:
+                    return None
+
+                # Sorteer op versienummer (major.minor)
+                available_versions.sort(key=lambda v: version.parse(v))
+                major_minor = available_versions[-1]
+
+            # Nu we major_minor hebben, zoek de hoogste patch
+            major_minor_path = os.path.join(content_path, major_minor)
+
+            if not os.path.exists(major_minor_path):
+                logger.error(f"Version path does not exist: {major_minor_path}")
+                return None
+
+            files = os.listdir(major_minor_path)
+            version_files = []
+
+            for file in files:
+                mm, p = self._parse_version(file)
+                if mm == major_minor and p:
+                    version_files.append((mm, p, f"{mm}.{p}"))
+
+            if not version_files:
+                return None
+
+            # Sorteer op patch nummer
+            version_files.sort(key=lambda v: int(v[1]))
+            return version_files[-1]
+
+        except Exception as e:
+            logger.error(f"Error finding latest version for {content_type}: {str(e)}")
+            return None
+
+    def read_content(self, content_type, major_minor=None, patch=None):
+        """
+        Lees content met versieondersteuning
+
+        Als major_minor en patch niet zijn opgegeven, wordt de laatste versie gebruikt.
+        Als alleen major_minor is opgegeven, wordt de laatste patch van die versie gebruikt.
+
+        Args:
+            content_type (str): Type content (bv. 'changelog', 'terms')
+            major_minor (str, optional): Major.Minor versie (bv. '1.0')
+            patch (str, optional): Patchnummer (bv. '5')
+
+        Returns:
+            dict: {
+                'content': str,
+                'version': str,
+                'content_type': str
+            } of None bij fout
+        """
+        try:
+            # Als geen versie opgegeven, vind de laatste
+            if not major_minor:
+                version_info = self.get_latest_version(content_type)
+                if not version_info:
+                    logger.error(f"No versions found for {content_type}")
+                    return None
+
+                major_minor, patch, full_version = version_info
+
+            # Als geen patch opgegeven, vind de laatste patch voor deze major_minor
+            elif not patch:
+                version_info = self.get_latest_version(content_type, major_minor)
+                if not version_info:
+                    logger.error(f"No versions found for {content_type} {major_minor}")
+                    return None
+
+                major_minor, patch, full_version = version_info
+            else:
+                full_version = f"{major_minor}.{patch}"
+
+            # Nu hebben we major_minor en patch, lees het bestand
+            file_path = self.get_content_path(content_type, major_minor, patch)
+
+            if not os.path.exists(file_path):
+                logger.error(f"Content file does not exist: {file_path}")
+                return None
+
+            with open(file_path, 'r', encoding='utf-8') as file:
+                content = file.read()
+
+            return {
+                'content': content,
+                'version': full_version,
+                'content_type': content_type
+            }
+
+        except Exception as e:
+            logger.error(f"Error reading content {content_type} {major_minor}.{patch}: {str(e)}")
+            return None
+
+    def list_content_types(self):
+        """Lijst alle beschikbare contenttypes op"""
+        try:
+            return [d for d in os.listdir(self.app.config['CONTENT_DIR']) 
+                   if os.path.isdir(os.path.join(self.app.config['CONTENT_DIR'], d))]
+        except Exception as e:
+            logger.error(f"Error listing content types: {str(e)}")
+            return []
+
+    def list_versions(self, content_type):
+        """
+        Lijst alle beschikbare versies voor een contenttype
+
+        Returns:
+            list: Lijst van dicts met versie-informatie
+                  [{'version': '1.0.0', 'path': '/path/to/file', 'date_modified': datetime}]
+        """
+        versions = []
+        try:
+            content_path = os.path.join(self.app.config['CONTENT_DIR'], content_type)
+
+            if not os.path.exists(content_path):
+                return []
+
+            for major_minor in os.listdir(content_path):
+                major_minor_path = os.path.join(content_path, major_minor)
+
+                if not os.path.isdir(major_minor_path):
+                    continue
+
+                for file in os.listdir(major_minor_path):
+                    mm, p = self._parse_version(file)
+                    if mm and p:
+                        file_path = os.path.join(major_minor_path, file)
+                        mod_time = os.path.getmtime(file_path)
+                        versions.append({
+                            'version': f"{mm}.{p}",
+                            'path': file_path,
+                            'date_modified': mod_time
+                        })
+
+            # Sorteer op versienummer
+            versions.sort(key=lambda v: version.parse(v['version']))
+            return versions
+
+        except Exception as e:
+            logger.error(f"Error listing versions for {content_type}: {str(e)}")
+            return []

common/utils/cors_utils.py

@@ -1,14 +1,14 @@
 from flask import request, current_app, session
+from flask_jwt_extended import decode_token, verify_jwt_in_request, get_jwt_identity
+
 from common.models.user import Tenant, TenantDomain
 
 
 def get_allowed_origins(tenant_id):
     session_key = f"allowed_origins_{tenant_id}"
     if session_key in session:
-        current_app.logger.debug(f"Fetching allowed origins for tenant {tenant_id} from session")
         return session[session_key]
 
-    current_app.logger.debug(f"Fetching allowed origins for tenant {tenant_id} from database")
     tenant_domains = TenantDomain.query.filter_by(tenant_id=int(tenant_id)).all()
     allowed_origins = [domain.domain for domain in tenant_domains]
 
@@ -18,51 +18,52 @@ def get_allowed_origins(tenant_id):
 
 
 def cors_after_request(response, prefix):
-    current_app.logger.debug(f'CORS after request: {request.path}, prefix: {prefix}')
-    current_app.logger.debug(f'request.headers: {request.headers}')
-    current_app.logger.debug(f'request.args: {request.args}')
-    current_app.logger.debug(f'request is json?: {request.is_json}')
-
     # Exclude health checks from checks
     if request.path.startswith('/healthz') or request.path.startswith('/_healthz'):
-        current_app.logger.debug('Skipping CORS headers for health checks')
         response.headers.add('Access-Control-Allow-Origin', '*')
         response.headers.add('Access-Control-Allow-Headers', '*')
         response.headers.add('Access-Control-Allow-Methods', '*')
         return response
 
+    # Handle OPTIONS preflight requests
+    if request.method == 'OPTIONS':
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization,X-Tenant-ID')
+        response.headers.add('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS')
+        response.headers.add('Access-Control-Allow-Credentials', 'true')
+        return response
+
     tenant_id = None
     allowed_origins = []
 
-    # Try to get tenant_id from JSON payload
-    json_data = request.get_json(silent=True)
-    current_app.logger.debug(f'request.get_json(silent=True): {json_data}')
-
-    if json_data and 'tenant_id' in json_data:
-        tenant_id = json_data['tenant_id']
+    # Check Socket.IO connection
+    if 'socket.io' in request.path:
+        token = request.args.get('token')
+        if token:
+            try:
+                decoded = decode_token(token)
+                tenant_id = decoded['sub']
+            except Exception as e:
+                current_app.logger.error(f'Error decoding token: {e}')
+                return response
     else:
-        # Fallback to get tenant_id from query parameters or headers if JSON is not available
-        tenant_id = request.args.get('tenant_id') or request.args.get('tenantId') or request.headers.get('X-Tenant-ID')
-
-    current_app.logger.debug(f'Identified tenant_id: {tenant_id}')
+        # Regular API requests
+        try:
+            if verify_jwt_in_request(optional=True):
+                tenant_id = get_jwt_identity()
+        except Exception as e:
+            current_app.logger.error(f'Error verifying JWT: {e}')
+            return response
 
     if tenant_id:
+        origin = request.headers.get('Origin')
         allowed_origins = get_allowed_origins(tenant_id)
-        current_app.logger.debug(f'Allowed origins for tenant {tenant_id}: {allowed_origins}')
-    else:
-        current_app.logger.warning('tenant_id not found in request')
 
-    origin = request.headers.get('Origin')
-    current_app.logger.debug(f'Origin: {origin}')
-
-    if origin in allowed_origins:
-        response.headers.add('Access-Control-Allow-Origin', origin)
-        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
-        response.headers.add('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS')
-        response.headers.add('Access-Control-Allow-Credentials', 'true')
-        current_app.logger.debug(f'CORS headers set for origin: {origin}')
-    else:
-        current_app.logger.warning(f'Origin {origin} not allowed')
+        if origin in allowed_origins:
+            response.headers.add('Access-Control-Allow-Origin', origin)
+            response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
+            response.headers.add('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS')
+            response.headers.add('Access-Control-Allow-Credentials', 'true')
 
     return response
 

common/utils/debug_utils.py

@@ -1,62 +1,104 @@
 from flask import request, session
 import time
 from flask_security import current_user
+import json
 
 
 def log_request_middleware(app):
-    # @app.before_request
-    # def log_request_info():
-    #     start_time = time.time()
-    #     app.logger.debug(f"Request URL: {request.url}")
-    #     app.logger.debug(f"Request Method: {request.method}")
-    #     app.logger.debug(f"Request Headers: {request.headers}")
-    #     app.logger.debug(f"Time taken for logging request info: {time.time() - start_time} seconds")
-    #     try:
-    #         app.logger.debug(f"Request Body: {request.get_data()}")
-    #     except Exception as e:
-    #         app.logger.error(f"Error reading request body: {e}")
-    #     app.logger.debug(f"Time taken for logging request body: {time.time() - start_time} seconds")
-
-    # @app.before_request
-    # def check_csrf_token():
-    #     start_time = time.time()
-    #     if request.method == "POST":
-    #         csrf_token = request.form.get("csrf_token")
-    #         app.logger.debug(f"CSRF Token: {csrf_token}")
-    #     app.logger.debug(f"Time taken for logging CSRF token: {time.time() - start_time} seconds")
-
-    # @app.before_request
-    # def log_user_info():
-    #     if current_user and current_user.is_authenticated:
-    #         app.logger.debug(f"Before: User ID: {current_user.id}")
-    #         app.logger.debug(f"Before: User Email: {current_user.email}")
-    #         app.logger.debug(f"Before: User Roles: {current_user.roles}")
-    #     else:
-    #         app.logger.debug("After: No user logged in")
 
     @app.before_request
     def log_session_state_before():
-        app.logger.debug(f'Session state before request: {session.items()}')
-
-    # @app.after_request
-    # def log_response_info(response):
-    #     start_time = time.time()
-    #     app.logger.debug(f"Response Status: {response.status}")
-    #     app.logger.debug(f"Response Headers: {response.headers}")
-    #
-    #     app.logger.debug(f"Time taken for logging response info: {time.time() - start_time} seconds")
-    #     return response
-
-    # @app.after_request
-    # def log_user_after_request(response):
-    #     if current_user and current_user.is_authenticated:
-    #         app.logger.debug(f"After: User ID: {current_user.id}")
-    #         app.logger.debug(f"after: User Email: {current_user.email}")
-    #         app.logger.debug(f"After: User Roles: {current_user.roles}")
-    #     else:
-    #         app.logger.debug("After: No user logged in")
+        pass
 
     @app.after_request
     def log_session_state_after(response):
-        app.logger.debug(f'Session state after request: {session.items()}')
         return response
+
+
+def register_request_debugger(app):
+    @app.before_request
+    def debug_request_info():
+        """Log consolidated request information for debugging"""
+        # Skip health check endpoints
+        if request.path.startswith('/_healthz') or request.path.startswith('/healthz'):
+            return
+
+        # Gather all request information in a structured way
+        debug_info = {
+            "basic_info": {
+                "method": request.method,
+                "path": request.path,
+                "content_type": request.content_type,
+                "content_length": request.content_length
+            },
+            "environment": {
+                "remote_addr": request.remote_addr,
+                "user_agent": str(request.user_agent)
+            }
+        }
+
+        # Add headers (excluding sensitive ones)
+        safe_headers = {k: v for k, v in request.headers.items()
+                       if k.lower() not in ('authorization', 'cookie', 'x-api-key')}
+        debug_info["headers"] = safe_headers
+
+        # Add authentication info (presence only)
+        auth_header = request.headers.get('Authorization', '')
+        debug_info["auth_info"] = {
+            "has_auth_header": bool(auth_header),
+            "auth_type": auth_header.split(' ')[0] if auth_header else None,
+            "token_length": len(auth_header.split(' ')[1]) if auth_header and len(auth_header.split(' ')) > 1 else 0,
+            "header_format": 'Valid format' if auth_header.startswith('Bearer ') else 'Invalid format',
+            "raw_header": auth_header[:10] + '...' if auth_header else None  # Show first 10 chars only
+        }
+
+        # Add request data based on type
+        if request.is_json:
+            try:
+                json_data = request.get_json()
+                if isinstance(json_data, dict):
+                    # Remove sensitive fields from logging
+                    safe_json = {k: v for k, v in json_data.items()
+                               if not any(sensitive in k.lower()
+                                        for sensitive in ['password', 'token', 'secret', 'key'])}
+                    debug_info["request_data"] = {
+                        "type": "json",
+                        "content": safe_json
+                    }
+            except Exception as e:
+                debug_info["request_data"] = {
+                    "type": "json",
+                    "error": str(e)
+                }
+        elif request.form:
+            safe_form = {k: v for k, v in request.form.items()
+                        if not any(sensitive in k.lower()
+                                 for sensitive in ['password', 'token', 'secret', 'key'])}
+            debug_info["request_data"] = {
+                "type": "form",
+                "content": safe_form
+            }
+
+        # Add file information if present
+        if request.files:
+            debug_info["files"] = {
+                name: {
+                    "filename": f.filename,
+                    "content_type": f.content_type,
+                    "content_length": f.content_length if hasattr(f, 'content_length') else None
+                }
+                for name, f in request.files.items()
+            }
+
+        # Add CORS information if present
+        cors_headers = {
+            "origin": request.headers.get('Origin'),
+            "request_method": request.headers.get('Access-Control-Request-Method'),
+            "request_headers": request.headers.get('Access-Control-Request-Headers')
+        }
+        if any(cors_headers.values()):
+            debug_info["cors"] = {k: v for k, v in cors_headers.items() if v is not None}
+
+        # Format the debug info as a pretty-printed JSON string with indentation
+        formatted_debug_info = json.dumps(debug_info, indent=2, sort_keys=True)
+

common/utils/document_utils.py

@@ -3,28 +3,63 @@ from datetime import datetime as dt, timezone as tz
 from sqlalchemy import desc
 from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.utils import secure_filename
-from common.models.document import Document, DocumentVersion
+from common.models.document import Document, DocumentVersion, Catalog
 from common.extensions import db, minio_client
 from common.utils.celery_utils import current_celery
 from flask import current_app
-from flask_security import current_user
 import requests
-from urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote, urlunparse, parse_qs
 import os
-from .eveai_exceptions import EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType
+
+from .config_field_types import normalize_json_field
+from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
+                               EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
+from ..models.user import Tenant
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+from common.services.entitlements import LicenseUsageServices
+
+MB_CONVERTOR = 1_048_576
+
+
+def get_file_size(file):
+    try:
+        # Als file een bytes object is of iets anders dat len() ondersteunt
+        file_size = len(file)
+    except TypeError:
+        # Als file een FileStorage object is
+        current_position = file.tell()
+        file.seek(0, os.SEEK_END)
+        file_size = file.tell()
+        file.seek(current_position)
+
+    return file_size
 
 
 def create_document_stack(api_input, file, filename, extension, tenant_id):
+    # Precheck if we can add a document to the stack
+
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file)/MB_CONVERTOR)
+
     # Create the Document
-    new_doc = create_document(api_input, filename, tenant_id)
+    catalog_id = int(api_input.get('catalog_id'))
+    catalog = Catalog.query.get(catalog_id)
+    if not catalog:
+        raise EveAIInvalidCatalog(tenant_id, catalog_id)
+    new_doc = create_document(api_input, filename, catalog_id)
     db.session.add(new_doc)
 
+    url = api_input.get('url', '')
+    if url != '':
+        url = cope_with_local_url(api_input.get('url', ''))
+
     # Create the DocumentVersion
-    new_doc_vers = create_version_for_document(new_doc,
-                                               api_input.get('url', ''),
+    new_doc_vers = create_version_for_document(new_doc, tenant_id,
+                                               url,
+                                               api_input.get('sub_file_type', ''),
                                                api_input.get('language', 'en'),
                                                api_input.get('user_context', ''),
                                                api_input.get('user_metadata'),
+                                               api_input.get('catalog_properties')
                                                )
     db.session.add(new_doc_vers)
 
@@ -44,7 +79,7 @@ def create_document_stack(api_input, file, filename, extension, tenant_id):
     return new_doc, new_doc_vers
 
 
-def create_document(form, filename, tenant_id):
+def create_document(form, filename, catalog_id):
     new_doc = Document()
     if form['name'] == '':
         new_doc.name = filename.rsplit('.', 1)[0]
@@ -55,13 +90,14 @@ def create_document(form, filename, tenant_id):
         new_doc.valid_from = form['valid_from']
     else:
         new_doc.valid_from = dt.now(tz.utc)
-    new_doc.tenant_id = tenant_id
+    new_doc.catalog_id = catalog_id
     set_logging_information(new_doc, dt.now(tz.utc))
 
     return new_doc
 
 
-def create_version_for_document(document, url, language, user_context, user_metadata):
+def create_version_for_document(document, tenant_id, url, sub_file_type,  language, user_context, user_metadata,
+                                catalog_properties):
     new_doc_vers = DocumentVersion()
     if url != '':
         new_doc_vers.url = url
@@ -75,7 +111,13 @@ def create_version_for_document(document, url, language, user_context, user_meta
         new_doc_vers.user_context = user_context
 
     if user_metadata != '' and user_metadata is not None:
-        new_doc_vers.user_metadata = user_metadata
+        new_doc_vers.user_metadata = normalize_json_field(user_metadata, "user_metadata")
+
+    if catalog_properties != '' and catalog_properties is not None:
+        new_doc_vers.catalog_properties = normalize_json_field(catalog_properties, "catalog_properties")
+
+    if sub_file_type != '':
+        new_doc_vers.sub_file_type = sub_file_type
 
     new_doc_vers.document = document
 
@@ -86,21 +128,23 @@ def create_version_for_document(document, url, language, user_context, user_meta
 
 def upload_file_for_version(doc_vers, file, extension, tenant_id):
     doc_vers.file_type = extension
-    doc_vers.file_name = doc_vers.calc_file_name()
-    doc_vers.file_location = doc_vers.calc_file_location()
 
     # Normally, the tenant bucket should exist. But let's be on the safe side if a migration took place.
     minio_client.create_tenant_bucket(tenant_id)
 
     try:
-        minio_client.upload_document_file(
+        bn, on, size = minio_client.upload_document_file(
             tenant_id,
             doc_vers.doc_id,
             doc_vers.language,
             doc_vers.id,
-            doc_vers.file_name,
+            f"{doc_vers.id}.{extension}",
             file
         )
+        doc_vers.bucket_name = bn
+        doc_vers.object_name = on
+        doc_vers.file_size = size / MB_CONVERTOR  # Convert bytes to MB
+
         db.session.commit()
         current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
                                 f'document version {doc_vers.id} while uploading file.')
@@ -111,35 +155,6 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
         raise
 
 
-def set_logging_information(obj, timestamp):
-    obj.created_at = timestamp
-    obj.updated_at = timestamp
-
-    user_id = get_current_user_id()
-    if user_id:
-        obj.created_by = user_id
-        obj.updated_by = user_id
-
-
-def update_logging_information(obj, timestamp):
-    obj.updated_at = timestamp
-
-    user_id = get_current_user_id()
-    if user_id:
-        obj.updated_by = user_id
-
-
-def get_current_user_id():
-    try:
-        if current_user and current_user.is_authenticated:
-            return current_user.id
-        else:
-            return None
-    except Exception:
-        # This will catch any errors if current_user is not available (e.g., in API context)
-        return None
-
-
 def get_extension_from_content_type(content_type):
     content_type_map = {
         'text/html': 'html',
@@ -153,6 +168,8 @@ def get_extension_from_content_type(content_type):
 
 
 def process_url(url, tenant_id):
+    url = cope_with_local_url(url)
+
     response = requests.head(url, allow_redirects=True)
     content_type = response.headers.get('Content-Type', '').split(';')[0]
 
@@ -184,57 +201,34 @@ def process_url(url, tenant_id):
     return file_content, filename, extension
 
 
-def process_multiple_urls(urls, tenant_id, api_input):
-    results = []
-    for url in urls:
-        try:
-            file_content, filename, extension = process_url(url, tenant_id)
+def clean_url(url):
+    tracking_params = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
+                       "hsa_acc", "hsa_cam", "hsa_grp", "hsa_ad", "hsa_src", "hsa_tgt", "hsa_kw",
+                       "hsa_mt", "hsa_net", "hsa_ver", "gad_source", "gbraid"}
 
-            url_input = api_input.copy()
-            url_input.update({
-                'url': url,
-                'name': f"{api_input['name']}-{filename}" if api_input['name'] else filename
-            })
+    parsed_url = urlparse(url)
+    query_params = parse_qs(parsed_url.query)
 
-            new_doc, new_doc_vers = create_document_stack(url_input, file_content, filename, extension, tenant_id)
-            task_id = start_embedding_task(tenant_id, new_doc_vers.id)
+    # Remove tracking params
+    clean_params = {k: v for k, v in query_params.items() if k not in tracking_params}
 
-            results.append({
-                'url': url,
-                'document_id': new_doc.id,
-                'document_version_id': new_doc_vers.id,
-                'task_id': task_id,
-                'status': 'success'
-            })
-        except Exception as e:
-            current_app.logger.error(f"Error processing URL {url}: {str(e)}")
-            results.append({
-                'url': url,
-                'status': 'error',
-                'message': str(e)
-            })
-    return results
+    # Reconstruct the URL
+    clean_query = "&".join(f"{k}={v[0]}" for k, v in clean_params.items()) if clean_params else ""
+    cleaned_url = urlunparse(parsed_url._replace(query=clean_query))
+
+    return cleaned_url
 
 
 def start_embedding_task(tenant_id, doc_vers_id):
-    task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
-        tenant_id,
-        doc_vers_id,
-    ])
+    task = current_celery.send_task('create_embeddings',
+                                    args=[tenant_id, doc_vers_id,],
+                                    queue='embeddings')
     current_app.logger.info(f'Embedding creation started for tenant {tenant_id}, '
                             f'Document Version {doc_vers_id}. '
                             f'Embedding creation task: {task.id}')
     return task.id
 
 
-def validate_file_type(extension):
-    current_app.logger.debug(f'Validating file type {extension}')
-    current_app.logger.debug(f'Supported file types: {current_app.config["SUPPORTED_FILE_TYPES"]}')
-    if extension not in current_app.config['SUPPORTED_FILE_TYPES']:
-        raise EveAIUnsupportedFileType(f"Filetype {extension} is currently not supported. "
-                                       f"Supported filetypes: {', '.join(current_app.config['SUPPORTED_FILE_TYPES'])}")
-
-
 def get_filename_from_url(url):
     parsed_url = urlparse(url)
     path_parts = parsed_url.path.split('/')
@@ -252,11 +246,16 @@ def get_documents_list(page, per_page):
     return pagination
 
 
-def edit_document(document_id, name, valid_from, valid_to):
-    doc = Document.query.get_or_404(document_id)
-    doc.name = name
-    doc.valid_from = valid_from
-    doc.valid_to = valid_to
+def edit_document(tenant_id, document_id, name, valid_from, valid_to):
+    doc = Document.query.get(document_id)
+    if not doc:
+        raise EveAIInvalidDocument(tenant_id, document_id)
+    if name:
+        doc.name = name
+    if valid_from:
+        doc.valid_from = valid_from
+    if valid_to:
+        doc.valid_to = valid_to
     update_logging_information(doc, dt.now(tz.utc))
 
     try:
@@ -268,9 +267,13 @@ def edit_document(document_id, name, valid_from, valid_to):
         return None, str(e)
 
 
-def edit_document_version(version_id, user_context):
-    doc_vers = DocumentVersion.query.get_or_404(version_id)
+def edit_document_version(tenant_id, version_id, user_context, catalog_properties):
+    doc_vers = DocumentVersion.query.get(version_id)
+    if not doc_vers:
+        raise EveAIInvalidDocumentVersion(tenant_id, version_id)
     doc_vers.user_context = user_context
+    doc_vers.catalog_properties = normalize_json_field(catalog_properties, "catalog_properties")
+
     update_logging_information(doc_vers, dt.now(tz.utc))
 
     try:
@@ -282,19 +285,25 @@ def edit_document_version(version_id, user_context):
         return None, str(e)
 
 
-def refresh_document_with_info(doc_id, api_input):
-    doc = Document.query.get_or_404(doc_id)
+def refresh_document_with_info(doc_id, tenant_id, api_input):
+    doc = Document.query.get(doc_id)
+    if not doc:
+        raise EveAIInvalidDocument(tenant_id, doc_id)
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
-
     if not old_doc_vers.url:
         return None, "This document has no URL. Only documents with a URL can be refreshed."
 
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, old_doc_vers.file_size)
+
     new_doc_vers = create_version_for_document(
-        doc,
+        doc, tenant_id,
         old_doc_vers.url,
+        old_doc_vers.sub_file_type,
         api_input.get('language', old_doc_vers.language),
         api_input.get('user_context', old_doc_vers.user_context),
-        api_input.get('user_metadata', old_doc_vers.user_metadata)
+        api_input.get('user_metadata', old_doc_vers.user_metadata),
+        api_input.get('catalog_properties', old_doc_vers.catalog_properties),
     )
 
     set_logging_information(new_doc_vers, dt.now(tz.utc))
@@ -306,33 +315,157 @@ def refresh_document_with_info(doc_id, api_input):
         db.session.rollback()
         return None, str(e)
 
-    response = requests.head(old_doc_vers.url, allow_redirects=True)
+    url = cope_with_local_url(old_doc_vers.url)
+    response = requests.head(url, allow_redirects=True)
     content_type = response.headers.get('Content-Type', '').split(';')[0]
     extension = get_extension_from_content_type(content_type)
 
-    response = requests.get(old_doc_vers.url)
+    response = requests.get(url)
     response.raise_for_status()
     file_content = response.content
 
-    upload_file_for_version(new_doc_vers, file_content, extension, doc.tenant_id)
+    upload_file_for_version(new_doc_vers, file_content, extension, tenant_id)
 
-    task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
-        doc.tenant_id,
-        new_doc_vers.id,
-    ])
+    task = current_celery.send_task('create_embeddings', args=[tenant_id, new_doc_vers.id,], queue='embeddings')
+    current_app.logger.info(f'Embedding creation started for document {doc_id} on version {new_doc_vers.id} '
+                            f'with task id: {task.id}.')
+
+    return new_doc_vers, task.id
+
+
+def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: bytes, api_input: dict) -> tuple:
+    """
+    Refresh document with new content
+
+    Args:
+        doc_id: Document ID
+        tenant_id: Tenant ID
+        file_content: New file content
+        api_input: Additional document information
+
+    Returns:
+        Tuple of (new_version, task_id)
+    """
+    doc = Document.query.get(doc_id)
+    if not doc:
+        raise EveAIInvalidDocument(tenant_id, doc_id)
+
+    old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
+
+    # Precheck if we have enough quota for the new version
+    LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file_content) / MB_CONVERTOR)
+
+    # Create new version with same file type as original
+    extension = old_doc_vers.file_type
+
+    new_doc_vers = create_version_for_document(
+        doc, tenant_id,
+        '',  # No URL for content-based updates
+        old_doc_vers.sub_file_type,
+        api_input.get('language', old_doc_vers.language),
+        api_input.get('user_context', old_doc_vers.user_context),
+        api_input.get('user_metadata', old_doc_vers.user_metadata),
+        api_input.get('catalog_properties', old_doc_vers.catalog_properties),
+    )
+
+    try:
+        db.session.add(new_doc_vers)
+        db.session.commit()
+    except SQLAlchemyError as e:
+        db.session.rollback()
+        return None, str(e)
+
+    # Upload new content
+    upload_file_for_version(new_doc_vers, file_content, extension, tenant_id)
+
+    # Start embedding task
+    task = current_celery.send_task('create_embeddings', args=[tenant_id, new_doc_vers.id], queue='embeddings')
+    current_app.logger.info(f'Embedding creation started for document {doc_id} on version {new_doc_vers.id} '
+                            f'with task id: {task.id}.')
 
     return new_doc_vers, task.id
 
 
 # Update the existing refresh_document function to use the new refresh_document_with_info
-def refresh_document(doc_id):
+def refresh_document(doc_id, tenant_id):
+    current_app.logger.info(f'Refreshing document {doc_id}')
     doc = Document.query.get_or_404(doc_id)
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
     api_input = {
         'language': old_doc_vers.language,
         'user_context': old_doc_vers.user_context,
-        'user_metadata': old_doc_vers.user_metadata
+        'user_metadata': old_doc_vers.user_metadata,
+        'catalog_properties': old_doc_vers.catalog_properties,
     }
 
-    return refresh_document_with_info(doc_id, api_input)
+    return refresh_document_with_info(doc_id, tenant_id, api_input)
+
+
+def cope_with_local_url(url):
+    parsed_url = urlparse(url)
+    # Check if this is an internal WordPress URL (TESTING) and rewrite it
+    if parsed_url.netloc in [current_app.config['EXTERNAL_WORDPRESS_BASE_URL']]:
+        parsed_url = parsed_url._replace(
+            scheme=current_app.config['WORDPRESS_PROTOCOL'],
+            netloc=f"{current_app.config['WORDPRESS_HOST']}:{current_app.config['WORDPRESS_PORT']}"
+        )
+        url = urlunparse(parsed_url)
+
+    return url
+
+
+def lookup_document(tenant_id: int, lookup_criteria: dict, metadata_type: str) -> tuple[Document, DocumentVersion]:
+    """
+    Look up a document using metadata criteria
+
+    Args:
+        tenant_id: ID of the tenant
+        lookup_criteria: Dictionary of key-value pairs to match in metadata
+        metadata_type: Which metadata to search in ('user_metadata' or 'system_metadata')
+
+    Returns:
+        Tuple of (Document, DocumentVersion) if found
+
+    Raises:
+        ValueError: If invalid metadata_type provided
+        EveAIException: If lookup fails
+    """
+    if metadata_type not in ['user_metadata', 'system_metadata']:
+        raise ValueError(f"Invalid metadata_type: {metadata_type}")
+
+    try:
+        # Query for the latest document version matching the criteria
+        query = (db.session.query(Document, DocumentVersion)
+                 .join(DocumentVersion)
+                 .filter(Document.id == DocumentVersion.doc_id)
+                 .order_by(DocumentVersion.id.desc()))
+
+        # Add metadata filtering using PostgreSQL JSONB operators
+        metadata_field = getattr(DocumentVersion, metadata_type)
+        for key, value in lookup_criteria.items():
+            query = query.filter(metadata_field[key].astext == str(value))
+
+        # Get first result
+        result = query.first()
+
+        if not result:
+            raise EveAIException(
+                f"No document found matching criteria in {metadata_type}",
+                status_code=404
+            )
+
+        return result
+
+    except SQLAlchemyError as e:
+        current_app.logger.error(f'Database error during document lookup for tenant {tenant_id}: {e}')
+        raise EveAIException(
+            "Database error during document lookup",
+            status_code=500
+        )
+    except Exception as e:
+        current_app.logger.error(f'Error during document lookup for tenant {tenant_id}: {e}')
+        raise EveAIException(
+            "Error during document lookup",
+            status_code=500
+        )

common/utils/dynamic_field_utils.py

@@ -0,0 +1,46 @@
+def create_default_config_from_type_config(type_config):
+    """
+    Creëert een dictionary met standaardwaarden gebaseerd op een typedefinitie configuratie.
+
+    Args:
+        type_config (dict): Het configuration-veld van een typedefinitie (bijv. uit processor_types).
+
+    Returns:
+        dict: Een dictionary met de naam van ieder veld als sleutel en de standaardwaarde als waarde.
+        Alleen velden met een standaardwaarde of die verplicht zijn, worden opgenomen.
+
+    Voorbeeld:
+        >>> config = PROCESSOR_TYPES["HTML_PROCESSOR"]["configuration"]
+        >>> create_default_config_from_type_def(config)
+        {'html_tags': 'p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td',
+         'html_end_tags': 'p, li, table',
+         'html_excluded_classes': '',
+         'html_excluded_elements': 'header, footer, nav, script',
+         'html_included_elements': 'article, main',
+         'chunking_heading_level': 2}
+    """
+    if not type_config:
+        return {}
+
+    default_config = {}
+
+    for field_name, field_def in type_config.items():
+        # Als het veld een standaardwaarde heeft, voeg deze toe
+        if "default" in field_def:
+            default_config[field_name] = field_def["default"]
+        # Als het veld verplicht is maar geen standaardwaarde heeft, voeg een lege string toe
+        elif field_def.get("required", False):
+            # Kies een geschikte "lege" waarde op basis van het type
+            field_type = field_def.get("type", "string")
+            if field_type == "string":
+                default_config[field_name] = ""
+            elif field_type == "integer":
+                default_config[field_name] = 0
+            elif field_type == "boolean":
+                default_config[field_name] = False
+            elif field_type == "color":
+                default_config[field_name] = "#000000"
+            else:
+                default_config[field_name] = ""
+
+    return default_config

common/utils/errors.py

@@ -0,0 +1,135 @@
+
+import traceback
+
+import jinja2
+from flask import render_template, request, jsonify, redirect, current_app, flash
+from flask_login import current_user
+
+from common.utils.eveai_exceptions import EveAINoSessionTenant
+from common.utils.nginx_utils import prefixed_url_for
+
+
+def not_found_error(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Not Found Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/404.html'), 404
+
+
+def internal_server_error(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Internal Server Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/500.html'), 500
+
+
+def not_authorised_error(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Not Authorised Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/401.html')
+
+
+def access_forbidden(error):
+    if not current_user.is_authenticated:
+        return redirect(prefixed_url_for('security.login'))
+    current_app.logger.error(f"Access Forbidden: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/403.html')
+
+
+def key_error_handler(error):
+    # Check if the KeyError is specifically for 'tenant'
+    if str(error) == "'tenant'":
+        return redirect(prefixed_url_for('security.login'))
+    # For other KeyErrors, you might want to log the error and return a generic error page
+    current_app.logger.error(f"Key Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/generic.html', error_message="An unexpected error occurred"), 500
+
+
+def attribute_error_handler(error):
+    """Handle AttributeError exceptions.
+
+            Specifically catches SQLAlchemy relationship errors when string IDs
+            are used instead of model instances.
+            """
+    error_msg = str(error)
+    current_app.logger.error(f"AttributeError: {error_msg}")
+    current_app.logger.error(traceback.format_exc())
+
+    # Handle the SQLAlchemy relationship error specifically
+    if "'str' object has no attribute '_sa_instance_state'" in error_msg:
+        flash('Database relationship error. Please check your form inputs and try again.', 'error')
+        return render_template('error/500.html',
+                               error_type="Relationship Error",
+                               error_details="A string value was provided where a database object was expected."), 500
+
+    # Handle other AttributeErrors
+    flash('An application error occurred. The technical team has been notified.', 'error')
+    return render_template('error/500.html',
+                           error_type="Attribute Error",
+                           error_details=error_msg), 500
+
+
+def no_tenant_selected_error(error):
+    """Handle errors when no tenant is selected in the current session.
+
+    This typically happens when a session expires or becomes invalid after
+    a long period of inactivity. The user will be redirected to the login page.
+    """
+    current_app.logger.error(f"No Session Tenant Error: {error}")
+    current_app.logger.error(traceback.format_exc())
+    flash('Your session expired. You will have to re-enter your credentials', 'warning')
+
+    # Perform logout if user is authenticated
+    if current_user.is_authenticated:
+        from flask_security.utils import logout_user
+        logout_user()
+
+    # Redirect to login page
+    return redirect(prefixed_url_for('security.login'))
+
+
+def general_exception(e):
+    current_app.logger.error(f"Unhandled Exception: {e}", exc_info=True)
+    flash('An application error occurred. The technical team has been notified.', 'error')
+    return render_template('error/500.html',
+                           error_type=type(e).__name__,
+                           error_details=str(e)), 500
+
+
+def template_not_found_error(error):
+    """Handle Jinja2 TemplateNotFound exceptions."""
+    current_app.logger.error(f'Template not found: {error.name}')
+    current_app.logger.error(f'Search Paths: {current_app.jinja_loader.list_templates()}')
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/500.html',
+                          error_type="Template Not Found",
+                          error_details=f"Template '{error.name}' could not be found."), 404
+
+
+def template_syntax_error(error):
+    """Handle Jinja2 TemplateSyntaxError exceptions."""
+    current_app.logger.error(f'Template syntax error: {error.message}')
+    current_app.logger.error(f'In template {error.filename}, line {error.lineno}')
+    current_app.logger.error(traceback.format_exc())
+    return render_template('error/500.html',
+                          error_type="Template Syntax Error",
+                          error_details=f"Error in template '{error.filename}' at line {error.lineno}: {error.message}"), 500
+
+
+def register_error_handlers(app):
+    app.register_error_handler(404, not_found_error)
+    app.register_error_handler(500, internal_server_error)
+    app.register_error_handler(401, not_authorised_error)
+    app.register_error_handler(403, not_authorised_error)
+    app.register_error_handler(EveAINoSessionTenant, no_tenant_selected_error)
+    app.register_error_handler(KeyError, key_error_handler)
+    app.register_error_handler(AttributeError, attribute_error_handler)
+    app.register_error_handler(jinja2.TemplateNotFound, template_not_found_error)
+    app.register_error_handler(jinja2.TemplateSyntaxError, template_syntax_error)
+    app.register_error_handler(Exception, general_exception)

common/utils/eveai_exceptions.py

@@ -10,8 +10,12 @@ class EveAIException(Exception):
     def to_dict(self):
         rv = dict(self.payload or ())
         rv['message'] = self.message
+        rv['error'] = self.__class__.__name__
         return rv
 
+    def __str__(self):
+        return self.message  # Return the message when the exception is converted to a string
+
 
 class EveAIInvalidLanguageException(EveAIException):
     """Raised when an invalid language is provided"""
@@ -34,3 +38,213 @@ class EveAIUnsupportedFileType(EveAIException):
         super().__init__(message, status_code, payload)
 
 
+class EveAINoLicenseForTenant(EveAIException):
+    """Raised when no active license for a tenant is provided"""
+
+    def __init__(self, message="No license for tenant found", status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAITenantNotFound(EveAIException):
+    """Raised when a tenant is not found"""
+
+    def __init__(self, tenant_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        message = f"Tenant {tenant_id} not found"
+        super().__init__(message, status_code, payload)
+
+
+class EveAITenantInvalid(EveAIException):
+    """Raised when a tenant is invalid"""
+
+    def __init__(self, tenant_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' is not valid. Please contact the System Administrator."
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoActiveLicense(EveAIException):
+    """Raised when a tenant has no active licenses"""
+
+    def __init__(self, tenant_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' has no active licenses. Please contact the System Administrator."
+        super().__init__(message, status_code, payload)
+
+
+class EveAIInvalidCatalog(EveAIException):
+    """Raised when a catalog cannot be found"""
+
+    def __init__(self, tenant_id, catalog_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        self.catalog_id = catalog_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' has no valid catalog with ID {catalog_id}. Please contact the System Administrator."
+        super().__init__(message, status_code, payload)
+
+
+class EveAIInvalidProcessor(EveAIException):
+    """Raised when no valid processor can be found for a given Catalog ID"""
+
+    def __init__(self, tenant_id, catalog_id, file_type, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        self.catalog_id = catalog_id
+        self.file_type = file_type
+        # Construct the message dynamically
+        message = (f"Tenant with ID '{tenant_id}' has no valid {file_type} processor for catalog with ID {catalog_id}. "
+                   f"Please contact the System Administrator.")
+        super().__init__(message, status_code, payload)
+
+
+class EveAIInvalidDocument(EveAIException):
+    """Raised when a tenant has no document with given ID"""
+
+    def __init__(self, tenant_id, document_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        self.document_id = document_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' has no document with ID {document_id}."
+        super().__init__(message, status_code, payload)
+
+
+class EveAIInvalidDocumentVersion(EveAIException):
+    """Raised when a tenant has no document version with given ID"""
+
+    def __init__(self, tenant_id, document_version_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        self.document_version_id = document_version_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' has no document version with ID {document_version_id}."
+        super().__init__(message, status_code, payload)
+
+
+class EveAISocketInputException(EveAIException):
+    """Raised when a socket call receives an invalid payload"""
+
+    def __init__(self, message, status_code=400, payload=None):
+        super.__init__(message, status_code, payload)
+
+
+class EveAIInvalidEmbeddingModel(EveAIException):
+    """Raised when no or an invalid embedding model is provided in the catalog"""
+
+    def __init__(self, tenant_id, catalog_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        self.catalog_id = catalog_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' has no or an invalid embedding model in Catalog {catalog_id}."
+        super().__init__(message, status_code, payload)
+
+
+class EveAIDoublePartner(EveAIException):
+    """Raised when there is already a partner defined for a given tenant (while registering a partner)"""
+
+    def __init__(self, tenant_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' is already defined as a Partner."
+        super().__init__(message, status_code, payload)
+
+
+class EveAIRoleAssignmentException(EveAIException):
+    """Exception raised when a role cannot be assigned due to business rules"""
+
+    def __init__(self, message, status_code=403, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoManagementPartnerService(EveAIException):
+    """Exception raised when the operation requires the logged in partner (or selected parter by Super User)
+    does not have a MANAGEMENT_SERVICE"""
+
+    def __init__(self, message="No Management Service defined for partner", status_code=403, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoSessionTenant(EveAIException):
+    """Exception raised when no session tenant is set"""
+
+    def __init__(self, message="No Session Tenant selected. Cannot perform requested action.", status_code=403,
+                 payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoSessionPartner(EveAIException):
+    """Exception raised when no session partner is set"""
+
+    def __init__(self, message="No Session Partner selected. Cannot perform requested action.", status_code=403,
+                 payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoManagementPartnerForTenant(EveAIException):
+    """Exception raised when the selected partner is no management partner for tenant"""
+
+    def __init__(self, message="No Management Partner for Tenant", status_code=403, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAIQuotaExceeded(EveAIException):
+    """Base exception for quota-related errors"""
+
+    def __init__(self, message, quota_type, current_usage, limit, additional=0, status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+        self.quota_type = quota_type
+        self.current_usage = current_usage
+        self.limit = limit
+        self.additional = additional
+
+
+class EveAIStorageQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when storage quota is exceeded"""
+
+    def __init__(self, current_usage, limit, additional, status_code=400, payload=None):
+        message = (f"Storage quota exceeded. Current: {current_usage:.1f}MB, "
+                   f"Additional: {additional:.1f}MB, Limit: {limit}MB")
+        super().__init__(message, "storage", current_usage, limit, additional, status_code, payload)
+
+
+class EveAIEmbeddingQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when embedding quota is exceeded"""
+
+    def __init__(self, current_usage, limit, additional, status_code=400, payload=None):
+        message = (f"Embedding quota exceeded. Current: {current_usage:.1f}MB, "
+                   f"Additional: {additional:.1f}MB, Limit: {limit}MB")
+        super().__init__(message, "embedding", current_usage, limit, additional, status_code, payload)
+
+
+class EveAIInteractionQuotaExceeded(EveAIQuotaExceeded):
+    """Raised when the interaction token quota is exceeded"""
+
+    def __init__(self, current_usage, limit, status_code=400, payload=None):
+        message = (f"Interaction token quota exceeded. Current: {current_usage:.2f}M tokens, "
+                   f"Limit: {limit:.2f}M tokens")
+        super().__init__(message, "interaction", current_usage, limit, 0, status_code, payload)
+
+
+class EveAIQuotaWarning(EveAIException):
+    """Warning when approaching quota limits (not blocking)"""
+
+    def __init__(self, message, quota_type, usage_percentage, status_code=200, payload=None):
+        super().__init__(message, status_code, payload)
+        self.quota_type = quota_type
+        self.usage_percentage = usage_percentage
+
+
+class EveAILicensePeriodsExceeded(EveAIException):
+    """Raised when no more license periods can be created for a given license"""
+
+    def __init__(self, license_id, status_code=400, payload=None):
+        message = f"No more license periods can be created for license with ID {license_id}. "
+        super().__init__(message, status_code, payload)
+
+
+class EveAIPendingLicensePeriod(EveAIException):
+    """Raised when a license period is pending"""
+
+    def __init__(self, status_code=400, payload=None):
+        message = f"Basic Fee Payment has not been received yet. Please ensure payment has been made, and please wait for payment to be processed."
+        super().__init__(message, status_code, payload)
+

common/utils/execution_progress.py

@@ -0,0 +1,112 @@
+# common/utils/execution_progress.py
+from datetime import datetime as dt, timezone as tz
+from typing import Generator
+from redis import Redis, RedisError
+import json
+from flask import current_app
+
+
+class ExecutionProgressTracker:
+    """Tracks progress of specialist executions using Redis"""
+
+    def __init__(self):
+        try:
+            redis_url = current_app.config['SPECIALIST_EXEC_PUBSUB']
+
+            self.redis = Redis.from_url(redis_url, socket_timeout=5)
+            # Test the connection
+            self.redis.ping()
+
+            self.expiry = 3600  # 1 hour expiry
+        except RedisError as e:
+            current_app.logger.error(f"Failed to connect to Redis: {str(e)}")
+            raise
+        except Exception as e:
+            current_app.logger.error(f"Unexpected error during Redis initialization: {str(e)}")
+            raise
+
+    def _get_key(self, execution_id: str) -> str:
+        return f"specialist_execution:{execution_id}"
+
+    def send_update(self, ctask_id: str, processing_type: str, data: dict):
+        """Send an update about execution progress"""
+        try:
+            key = self._get_key(ctask_id)
+
+            # First verify Redis is still connected
+            try:
+                self.redis.ping()
+            except RedisError:
+                current_app.logger.error("Lost Redis connection. Attempting to reconnect...")
+                self.__init__()  # Reinitialize connection
+
+            update = {
+                'processing_type': processing_type,
+                'data': data,
+                'timestamp': dt.now(tz=tz.utc)
+            }
+
+            # Log initial state
+            try:
+                orig_len = self.redis.llen(key)
+
+                # Try to serialize the update and check the result
+                try:
+                    serialized_update = json.dumps(update, default=str)  # Add default handler for datetime
+                except TypeError as e:
+                    current_app.logger.error(f"Failed to serialize update: {str(e)}")
+                    raise
+
+                # Store update in list with pipeline for atomicity
+                with self.redis.pipeline() as pipe:
+                    pipe.rpush(key, serialized_update)
+                    pipe.publish(key, serialized_update)
+                    pipe.expire(key, self.expiry)
+                    results = pipe.execute()
+
+                new_len = self.redis.llen(key)
+
+                if new_len <= orig_len:
+                    current_app.logger.error(
+                        f"List length did not increase as expected. Original: {orig_len}, New: {new_len}")
+
+            except RedisError as e:
+                current_app.logger.error(f"Redis operation failed: {str(e)}")
+                raise
+
+        except Exception as e:
+            current_app.logger.error(f"Unexpected error in send_update: {str(e)}, type: {type(e)}")
+            raise
+
+    def get_updates(self, ctask_id: str) -> Generator[str, None, None]:
+        key = self._get_key(ctask_id)
+        pubsub = self.redis.pubsub()
+        pubsub.subscribe(key)
+
+        try:
+            # First yield any existing updates
+            length = self.redis.llen(key)
+            if length > 0:
+                updates = self.redis.lrange(key, 0, -1)
+                for update in updates:
+                    update_data = json.loads(update.decode('utf-8'))
+                    # Use processing_type for the event
+                    yield f"event: {update_data['processing_type']}\n"
+                    yield f"data: {json.dumps(update_data)}\n\n"
+
+            # Then listen for new updates
+            while True:
+                message = pubsub.get_message(timeout=30)  # message['type'] is Redis pub/sub type
+                if message is None:
+                    yield ": keepalive\n\n"
+                    continue
+
+                if message['type'] == 'message':  # This is Redis pub/sub type
+                    update_data = json.loads(message['data'].decode('utf-8'))
+                    yield f"data: {message['data'].decode('utf-8')}\n\n"
+
+                    # Check processing_type for completion
+                    if update_data['processing_type'] in ['Task Complete', 'Task Error']:
+                        break
+        finally:
+            pubsub.unsubscribe()

common/utils/form_assistants.py

@@ -0,0 +1,11 @@
+import json
+
+from wtforms.validators import ValidationError
+
+
+def validate_json(form, field):
+    if field.data:
+        try:
+            json.loads(field.data)
+        except json.JSONDecodeError:
+            raise ValidationError('Invalid JSON format')

common/utils/log_utils.py

@@ -0,0 +1,57 @@
+import pandas as pd
+from sqlalchemy import inspect
+from typing import Any, List, Union, Optional
+
+
+def format_query_results(query_results: Any) -> str:
+    """
+    Format query results as a readable string using pandas
+
+    Args:
+        query_results: SQLAlchemy query, query results, or model instance(s)
+
+    Returns:
+        Formatted string representation of the query results
+    """
+    try:
+        # If it's a query object, execute it
+        if hasattr(query_results, 'all'):
+            results = query_results.all()
+        elif not isinstance(query_results, list):
+            results = [query_results]
+        else:
+            results = query_results
+
+        # Handle different types of results
+        if results and hasattr(results[0], '__table__'):
+            # SQLAlchemy ORM objects
+            data = []
+            for item in results:
+                row = {}
+                for column in inspect(item).mapper.column_attrs:
+                    row[column.key] = getattr(item, column.key)
+                data.append(row)
+            df = pd.DataFrame(data)
+        elif results and isinstance(results[0], tuple):
+            # Join query results (tuples)
+            if hasattr(results[0], '_fields'):  # Named tuples
+                df = pd.DataFrame(results)
+            else:
+                # Regular tuples - try to get column names from query
+                if hasattr(query_results, 'statement'):
+                    columns = query_results.statement.columns.keys()
+                    df = pd.DataFrame(results, columns=columns)
+                else:
+                    df = pd.DataFrame(results)
+        else:
+            # Fallback for other types
+            df = pd.DataFrame(results)
+
+        # Format the output with pandas
+        with pd.option_context('display.max_rows', 20, 'display.max_columns', None,
+                               'display.width', 1000):
+            formatted_output = f"Query returned {len(df)} results:\n{df}"
+
+        return formatted_output
+    except Exception as e:
+        return f"Error formatting query results: {str(e)}"

common/utils/mail_utils.py

@@ -0,0 +1,46 @@
+from scaleway import Client
+from scaleway.tem.v1alpha1.api import TemV1Alpha1API
+from scaleway.tem.v1alpha1.types import CreateEmailRequestAddress
+from html2text import HTML2Text
+from flask import current_app
+
+
+def send_email(to_email, to_name, subject, html):
+    current_app.logger.debug(f"Sending email to {to_email} with subject {subject}")
+    access_key = current_app.config['SW_EMAIL_ACCESS_KEY']
+    secret_key = current_app.config['SW_EMAIL_SECRET_KEY']
+    default_project_id = current_app.config['SW_PROJECT']
+    default_region = "fr-par"
+    current_app.logger.debug(f"Access Key: {access_key}\nSecret Key: {secret_key}\n"
+                             f"Default Project ID: {default_project_id}\nDefault Region: {default_region}")
+    client = Client(
+        access_key=access_key,
+        secret_key=secret_key,
+        default_project_id=default_project_id,
+        default_region=default_region
+    )
+    current_app.logger.debug(f"Scaleway Client Initialized")
+    tem = TemV1Alpha1API(client)
+    current_app.logger.debug(f"Tem Initialized")
+    from_ = CreateEmailRequestAddress(email=current_app.config['SW_EMAIL_SENDER'],
+                                      name=current_app.config['SW_EMAIL_NAME'])
+    to_ = CreateEmailRequestAddress(email=to_email, name=to_name)
+
+    email = tem.create_email(
+        from_=from_,
+        to=[to_],
+        subject=subject,
+        text=html_to_text(html),
+        html=html,
+        project_id=default_project_id,
+    )
+    current_app.logger.debug(f"Email sent to {to_email}")
+
+
+def html_to_text(html_content):
+    """Convert HTML to plain text using html2text"""
+    h = HTML2Text()
+    h.ignore_images = True
+    h.ignore_emphasis = False
+    h.body_width = 0  # No wrapping
+    return h.handle(html_content)

common/utils/middleware.py

@@ -4,10 +4,11 @@ for handling tenant requests
 """
 
 from flask_security import current_user
-from flask import session, current_app, redirect
-from common.utils.nginx_utils import prefixed_url_for
-
+from flask import session
 from .database import Database
+from .eveai_exceptions import EveAINoSessionTenant, EveAINoSessionPartner, EveAINoManagementPartnerService, \
+    EveAINoManagementPartnerForTenant
+from common.services.user import UserServices
 
 
 def mw_before_request():
@@ -17,20 +18,27 @@ def mw_before_request():
     """
 
     if 'tenant' not in session:
-        current_app.logger.warning('No tenant defined in session')
-        return redirect(prefixed_url_for('security_bp.login'))
+        raise EveAINoSessionTenant()
 
     tenant_id = session['tenant']['id']
     if not tenant_id:
-        raise Exception('Cannot switch schema for tenant: no tenant defined in session')
+        raise EveAINoSessionTenant()
 
-    for role in current_user.roles:
-        current_app.logger.debug(f'In middleware: User {current_user.email} has role {role.name}')
+    switch_allowed = False
+    if current_user.has_role('Super User'):
+        switch_allowed = True
+    if current_user.has_role('Tenant Admin') and current_user.tenant_id == tenant_id:
+        switch_allowed = True
+    if current_user.has_role('Partner Admin'):
+        if 'partner' not in session:
+            raise EveAINoSessionPartner()
+        management_service = next((service for service in session['partner']['services']
+                                   if service.get('type') == 'MANAGEMENT_SERVICE'), None)
+        if not management_service:
+            raise EveAINoManagementPartnerService()
+        if not UserServices.can_user_edit_tenant(tenant_id):
+            raise EveAINoManagementPartnerForTenant()
 
-    # user = User.query.get(current_user.id)
-    if current_user.has_role('Super User') or current_user.tenant_id == tenant_id:
-        Database(tenant_id).switch_schema()
-    else:
-        raise Exception(f'Cannot switch schema for tenant {tenant_id}: user {current_user.email} does not have access')
+    Database(tenant_id).switch_schema()
 
 

common/utils/minio_utils.py

@@ -33,6 +33,9 @@ class MinioClient:
     def generate_object_name(self, document_id, language, version_id, filename):
         return f"{document_id}/{language}/{version_id}/{filename}"
 
+    def generate_asset_name(self, asset_version_id, file_name, content_type):
+        return f"assets/{asset_version_id}/{file_name}.{content_type}"
+
     def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data):
         bucket_name = self.generate_bucket_name(tenant_id)
         object_name = self.generate_object_name(document_id, language, version_id, filename)
@@ -50,13 +53,31 @@ class MinioClient:
             self.client.put_object(
                 bucket_name, object_name, io.BytesIO(file_data), len(file_data)
             )
-            return True
+            return bucket_name, object_name, len(file_data)
         except S3Error as err:
             raise Exception(f"Error occurred while uploading file: {err}")
 
-    def download_document_file(self, tenant_id, document_id, language, version_id, filename):
-        bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_object_name(document_id, language, version_id, filename)
+    def upload_asset_file(self, bucket_name, asset_version_id, file_name, file_type, file_data):
+        object_name = self.generate_asset_name(asset_version_id, file_name, file_type)
+
+        try:
+            if isinstance(file_data, FileStorage):
+                file_data = file_data.read()
+            elif isinstance(file_data, io.BytesIO):
+                file_data = file_data.getvalue()
+            elif isinstance(file_data, str):
+                file_data = file_data.encode('utf-8')
+            elif not isinstance(file_data, bytes):
+                raise TypeError('Unsupported file type. Expected FileStorage, BytesIO, str, or bytes.')
+
+            self.client.put_object(
+                bucket_name, object_name, io.BytesIO(file_data), len(file_data)
+            )
+            return object_name, len(file_data)
+        except S3Error as err:
+            raise Exception(f"Error occurred while uploading asset: {err}")
+
+    def download_document_file(self, tenant_id, bucket_name, object_name):
         try:
             response = self.client.get_object(bucket_name, object_name)
             return response.read()

common/utils/model_logging_utils.py

@@ -0,0 +1,29 @@
+from flask_security import current_user
+
+
+def set_logging_information(obj, timestamp):
+    obj.created_at = timestamp
+    obj.updated_at = timestamp
+
+    user_id = get_current_user_id()
+    if user_id:
+        obj.created_by = user_id
+        obj.updated_by = user_id
+
+def update_logging_information(obj, timestamp):
+    obj.updated_at = timestamp
+
+    user_id = get_current_user_id()
+    if user_id:
+        obj.updated_by = user_id
+
+
+def get_current_user_id():
+    try:
+        if current_user and current_user.is_authenticated:
+            return current_user.id
+        else:
+            return None
+    except Exception:
+        # This will catch any errors if current_user is not available (e.g., in API context)
+        return None

common/utils/model_utils.py

@@ -1,305 +1,40 @@
 import os
+from typing import Dict, Any, Optional, Tuple
 
 import langcodes
-from flask import current_app
-from langchain_openai import OpenAIEmbeddings, ChatOpenAI
-from langchain_anthropic import ChatAnthropic
-from langchain_core.pydantic_v1 import BaseModel, Field
-from typing import List, Any, Iterator
-from collections.abc import MutableMapping
-from openai import OpenAI
-from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL
-from portkey_ai.langchain.portkey_langchain_callback_handler import LangchainCallbackHandler
+from langchain_core.language_models import BaseChatModel
 
-from common.models.document import EmbeddingSmallOpenAI, EmbeddingLargeOpenAI
+from common.langchain.llm_metrics_handler import LLMMetricsHandler
+from langchain_openai import ChatOpenAI
+from langchain_anthropic import ChatAnthropic
+from langchain_mistralai import ChatMistralAI
+from flask import current_app
+
+from common.eveai_model.tracked_mistral_embeddings import TrackedMistralAIEmbeddings
+from common.langchain.tracked_transcription import TrackedOpenAITranscription
 from common.models.user import Tenant
 from config.model_config import MODEL_CONFIG
-from common.utils.business_event_context import current_event
+from common.extensions import cache_manager
+from common.models.document import EmbeddingMistral
+from common.utils.eveai_exceptions import EveAITenantNotFound, EveAIInvalidEmbeddingModel
+from crewai import LLM
+
+embedding_llm_model_cache: Dict[Tuple[str, float], BaseChatModel] = {}
+crewai_llm_model_cache: Dict[Tuple[str, float], LLM] = {}
+llm_metrics_handler = LLMMetricsHandler()
 
 
-class CitedAnswer(BaseModel):
-    """Default docstring - to be replaced with actual prompt"""
+def create_language_template(template: str, language: str) -> str:
+    """
+    Replace language placeholder in template with specified language
 
-    answer: str = Field(
-        ...,
-        description="The answer to the user question, based on the given sources",
-    )
-    citations: List[int] = Field(
-        ...,
-        description="The integer IDs of the SPECIFIC sources that were used to generate the answer"
-    )
-    insufficient_info: bool = Field(
-        False,  # Default value is set to False
-        description="A boolean indicating wether given sources were sufficient or not to generate the answer"
-    )
+    Args:
+        template: Template string with {language} placeholder
+        language: Language code to insert
 
-
-def set_language_prompt_template(cls, language_prompt):
-    cls.__doc__ = language_prompt
-
-
-class ModelVariables(MutableMapping):
-    def __init__(self, tenant: Tenant):
-        self.tenant = tenant
-        self._variables = self._initialize_variables()
-        self._embedding_model = None
-        self._llm = None
-        self._llm_no_rag = None
-        self._transcription_client = None
-        self._prompt_templates = {}
-        self._embedding_db_model = None
-
-    def _initialize_variables(self):
-        variables = {}
-
-        # We initialize the variables that are available knowing the tenant. For the other, we will apply 'lazy loading'
-        variables['k'] = self.tenant.es_k or 5
-        variables['similarity_threshold'] = self.tenant.es_similarity_threshold or 0.7
-        variables['RAG_temperature'] = self.tenant.chat_RAG_temperature or 0.3
-        variables['no_RAG_temperature'] = self.tenant.chat_no_RAG_temperature or 0.5
-        variables['embed_tuning'] = self.tenant.embed_tuning or False
-        variables['rag_tuning'] = self.tenant.rag_tuning or False
-        variables['rag_context'] = self.tenant.rag_context or " "
-
-        # Set HTML Chunking Variables
-        variables['html_tags'] = self.tenant.html_tags
-        variables['html_end_tags'] = self.tenant.html_end_tags
-        variables['html_included_elements'] = self.tenant.html_included_elements
-        variables['html_excluded_elements'] = self.tenant.html_excluded_elements
-        variables['html_excluded_classes'] = self.tenant.html_excluded_classes
-
-        # Set Chunk Size variables
-        variables['min_chunk_size'] = self.tenant.min_chunk_size
-        variables['max_chunk_size'] = self.tenant.max_chunk_size
-
-        # Set model providers
-        variables['embedding_provider'], variables['embedding_model'] = self.tenant.embedding_model.rsplit('.', 1)
-        variables['llm_provider'], variables['llm_model'] = self.tenant.llm_model.rsplit('.', 1)
-        variables["templates"] = current_app.config['PROMPT_TEMPLATES'][(f"{variables['llm_provider']}."
-                                                                         f"{variables['llm_model']}")]
-        current_app.logger.info(f"Loaded prompt templates: \n")
-        current_app.logger.info(f"{variables['templates']}")
-
-        # Set model-specific configurations
-        model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {})
-        variables.update(model_config)
-
-        variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][self.tenant.llm_model]
-
-        if variables['tool_calling_supported']:
-            variables['cited_answer_cls'] = CitedAnswer
-
-        return variables
-
-    @property
-    def embedding_model(self):
-        portkey_metadata = self.get_portkey_metadata()
-
-        portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
-                                        provider=self._variables['embedding_provider'],
-                                        metadata=portkey_metadata,
-                                        trace_id=current_event.trace_id,
-                                        span_id=current_event.span_id,
-                                        span_name=current_event.span_name,
-                                        parent_span_id=current_event.parent_span_id
-                                        )
-        api_key = os.getenv('OPENAI_API_KEY')
-        model = self._variables['embedding_model']
-        self._embedding_model = OpenAIEmbeddings(api_key=api_key,
-                                                 model=model,
-                                                 base_url=PORTKEY_GATEWAY_URL,
-                                                 default_headers=portkey_headers)
-        self._embedding_db_model = EmbeddingSmallOpenAI \
-            if model == 'text-embedding-3-small' \
-            else EmbeddingLargeOpenAI
-
-        return self._embedding_model
-
-    @property
-    def llm(self):
-        portkey_headers = self.get_portkey_headers_for_llm()
-        api_key = self.get_api_key_for_llm()
-        self._llm = ChatOpenAI(api_key=api_key,
-                               model=self._variables['llm_model'],
-                               temperature=self._variables['RAG_temperature'],
-                               base_url=PORTKEY_GATEWAY_URL,
-                               default_headers=portkey_headers)
-        return self._llm
-
-    @property
-    def llm_no_rag(self):
-        portkey_headers = self.get_portkey_headers_for_llm()
-        api_key = self.get_api_key_for_llm()
-        self._llm_no_rag = ChatOpenAI(api_key=api_key,
-                                      model=self._variables['llm_model'],
-                                      temperature=self._variables['RAG_temperature'],
-                                      base_url=PORTKEY_GATEWAY_URL,
-                                      default_headers=portkey_headers)
-        return self._llm_no_rag
-
-    def get_portkey_headers_for_llm(self):
-        portkey_metadata = self.get_portkey_metadata()
-        portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
-                                        metadata=portkey_metadata,
-                                        provider=self._variables['llm_provider'],
-                                        trace_id=current_event.trace_id,
-                                        span_id=current_event.span_id,
-                                        span_name=current_event.span_name,
-                                        parent_span_id=current_event.parent_span_id
-                                        )
-        return portkey_headers
-
-    def get_portkey_metadata(self):
-        environment = os.getenv('FLASK_ENV', 'development')
-        portkey_metadata = {'tenant_id': str(self.tenant.id),
-                            'environment': environment,
-                            'trace_id': current_event.trace_id,
-                            'span_id': current_event.span_id,
-                            'span_name': current_event.span_name,
-                            'parent_span_id': current_event.parent_span_id,
-                            }
-        return portkey_metadata
-
-    def get_api_key_for_llm(self):
-        if self._variables['llm_provider'] == 'openai':
-            api_key = os.getenv('OPENAI_API_KEY')
-        else: # self._variables['llm_provider'] == 'anthropic'
-            api_key = os.getenv('ANTHROPIC_API_KEY')
-
-        return api_key
-
-    # def _initialize_llm(self):
-    #
-    #
-    #     if self._variables['llm_provider'] == 'openai':
-    #         portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
-    #                                         metadata=portkey_metadata,
-    #                                         provider='openai')
-    #
-    #         self._llm = ChatOpenAI(api_key=api_key,
-    #                                model=self._variables['llm_model'],
-    #                                temperature=self._variables['RAG_temperature'],
-    #                                base_url=PORTKEY_GATEWAY_URL,
-    #                                default_headers=portkey_headers)
-    #         self._llm_no_rag = ChatOpenAI(api_key=api_key,
-    #                                       model=self._variables['llm_model'],
-    #                                       temperature=self._variables['no_RAG_temperature'],
-    #                                       base_url=PORTKEY_GATEWAY_URL,
-    #                                       default_headers=portkey_headers)
-    #         self._variables['tool_calling_supported'] = self._variables['llm_model'] in ['gpt-4o', 'gpt-4o-mini']
-    #     elif self._variables['llm_provider'] == 'anthropic':
-    #         api_key = os.getenv('ANTHROPIC_API_KEY')
-    #         llm_model_ext = os.getenv('ANTHROPIC_LLM_VERSIONS', {}).get(self._variables['llm_model'])
-    #         self._llm = ChatAnthropic(api_key=api_key,
-    #                                   model=llm_model_ext,
-    #                                   temperature=self._variables['RAG_temperature'])
-    #         self._llm_no_rag = ChatAnthropic(api_key=api_key,
-    #                                          model=llm_model_ext,
-    #                                          temperature=self._variables['RAG_temperature'])
-    #         self._variables['tool_calling_supported'] = True
-    #     else:
-    #         raise ValueError(f"Invalid chat provider: {self._variables['llm_provider']}")
-
-    @property
-    def transcription_client(self):
-        environment = os.getenv('FLASK_ENV', 'development')
-        portkey_metadata = self.get_portkey_metadata()
-        portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
-                                        metadata=portkey_metadata,
-                                        provider='openai',
-                                        trace_id=current_event.trace_id,
-                                        span_id=current_event.span_id,
-                                        span_name=current_event.span_name,
-                                        parent_span_id=current_event.parent_span_id
-                                        )
-        api_key = os.getenv('OPENAI_API_KEY')
-        self._transcription_client = OpenAI(api_key=api_key,
-                                            base_url=PORTKEY_GATEWAY_URL,
-                                            default_headers=portkey_headers)
-        self._variables['transcription_model'] = 'whisper-1'
-        return self._transcription_client
-
-    @property
-    def embedding_db_model(self):
-        if self._embedding_db_model is None:
-            self._embedding_db_model = self.get_embedding_db_model()
-        return self._embedding_db_model
-
-    def get_embedding_db_model(self):
-        current_app.logger.debug("In get_embedding_db_model")
-        if self._embedding_db_model is None:
-            self._embedding_db_model = EmbeddingSmallOpenAI \
-                if self._variables['embedding_model'] == 'text-embedding-3-small' \
-                else EmbeddingLargeOpenAI
-        current_app.logger.debug(f"Embedding DB Model: {self._embedding_db_model}")
-        return self._embedding_db_model
-
-    def get_prompt_template(self, template_name: str) -> str:
-        current_app.logger.info(f"Getting prompt template for {template_name}")
-        if template_name not in self._prompt_templates:
-            self._prompt_templates[template_name] = self._load_prompt_template(template_name)
-        return self._prompt_templates[template_name]
-
-    def _load_prompt_template(self, template_name: str) -> str:
-        # In the future, this method will make an API call to Portkey
-        # For now, we'll simulate it with a placeholder implementation
-        # You can replace this with your current prompt loading logic
-        return self._variables['templates'][template_name]
-
-    def __getitem__(self, key: str) -> Any:
-        current_app.logger.debug(f"ModelVariables: Getting {key}")
-        # Support older template names (suffix = _template)
-        if key.endswith('_template'):
-            key = key[:-len('_template')]
-            current_app.logger.debug(f"ModelVariables: Getting modified {key}")
-        if key == 'embedding_model':
-            return self.embedding_model
-        elif key == 'embedding_db_model':
-            return self.embedding_db_model
-        elif key == 'llm':
-            return self.llm
-        elif key == 'llm_no_rag':
-            return self.llm_no_rag
-        elif key == 'transcription_client':
-            return self.transcription_client
-        elif key in self._variables.get('prompt_templates', []):
-            return self.get_prompt_template(key)
-        return self._variables.get(key)
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        self._variables[key] = value
-
-    def __delitem__(self, key: str) -> None:
-        del self._variables[key]
-
-    def __iter__(self) -> Iterator[str]:
-        return iter(self._variables)
-
-    def __len__(self):
-        return len(self._variables)
-
-    def get(self, key: str, default: Any = None) -> Any:
-        return self.__getitem__(key) or default
-
-    def update(self, **kwargs) -> None:
-        self._variables.update(kwargs)
-
-    def items(self):
-        return self._variables.items()
-
-    def keys(self):
-        return self._variables.keys()
-
-    def values(self):
-        return self._variables.values()
-
-
-def select_model_variables(tenant):
-    model_variables = ModelVariables(tenant=tenant)
-    return model_variables
-
-
-def create_language_template(template, language):
+    Returns:
+        str: Template with language placeholder replaced
+    """
     try:
         full_language = langcodes.Language.make(language=language)
         language_template = template.replace('{language}', full_language.display_name())
@@ -309,5 +44,121 @@ def create_language_template(template, language):
     return language_template
 
 
-def replace_variable_in_template(template, variable, value):
-    return template.replace(variable, value)
+def replace_variable_in_template(template: str, variable: str, value: str) -> str:
+    """
+    Replace a variable placeholder in template with specified value
+
+    Args:
+        template: Template string with variable placeholder
+        variable: Variable placeholder to replace (e.g. "{tenant_context}")
+        value: Value to insert
+
+    Returns:
+        str: Template with variable placeholder replaced
+    """
+    return template.replace(variable, value or "")
+
+
+def get_embedding_model_and_class(tenant_id, catalog_id, full_embedding_name="mistral.mistral-embed"):
+    """
+        Retrieve the embedding model and embedding model class to store Embeddings
+
+        Args:
+            tenant_id: ID of the tenant
+            catalog_id: ID of the catalog
+            full_embedding_name: The full name of the embedding model: <provider>.<model>
+
+        Returns:
+            embedding_model, embedding_model_class
+        """
+    embedding_provider, embedding_model_name = full_embedding_name.split('.')
+
+    # Calculate the embedding model to be used
+    if embedding_provider == "mistral":
+        api_key = current_app.config['MISTRAL_API_KEY']
+        embedding_model = TrackedMistralAIEmbeddings(
+            model=embedding_model_name
+        )
+    else:
+        raise EveAIInvalidEmbeddingModel(tenant_id, catalog_id)
+
+    # Calculate the Embedding Model Class to be used to store embeddings
+    if embedding_model_name == "mistral-embed":
+        embedding_model_class = EmbeddingMistral
+    else:
+        raise EveAIInvalidEmbeddingModel(tenant_id, catalog_id)
+
+    return embedding_model, embedding_model_class
+
+
+def get_embedding_llm(full_model_name='mistral.mistral-small-latest', temperature=0.3):
+    llm = embedding_llm_model_cache.get((full_model_name, temperature))
+    if not llm:
+        llm_provider, llm_model_name = full_model_name.split('.')
+        if llm_provider == "openai":
+            llm = ChatOpenAI(
+                api_key=current_app.config['OPENAI_API_KEY'],
+                model=llm_model_name,
+                temperature=temperature,
+                callbacks=[llm_metrics_handler]
+            )
+        elif llm_provider == "mistral":
+            llm = ChatMistralAI(
+                api_key=current_app.config['MISTRAL_API_KEY'],
+                model=llm_model_name,
+                temperature=temperature,
+                callbacks=[llm_metrics_handler]
+            )
+        embedding_llm_model_cache[(full_model_name, temperature)] = llm
+
+    return llm
+
+
+def get_crewai_llm(full_model_name='mistral.mistral-large-latest', temperature=0.3):
+    llm = crewai_llm_model_cache.get((full_model_name, temperature))
+    if not llm:
+        llm_provider, llm_model_name = full_model_name.split('.')
+        crew_full_model_name = f"{llm_provider}/{llm_model_name}"
+        api_key = None
+        if llm_provider == "openai":
+            api_key = current_app.config['OPENAI_API_KEY']
+        elif llm_provider == "mistral":
+            api_key = current_app.config['MISTRAL_API_KEY']
+
+        llm = LLM(
+            model=crew_full_model_name,
+            temperature=temperature,
+            api_key=api_key
+        )
+        crewai_llm_model_cache[(full_model_name, temperature)] = llm
+
+    return llm
+
+
+def process_pdf():
+    full_model_name = 'mistral-ocr-latest'
+
+
+def get_template(template_name: str, version: Optional[str] = "1.0", temperature: float = 0.3) -> tuple[
+    Any, BaseChatModel | None | ChatOpenAI | ChatMistralAI]:
+    """
+    Get a prompt template
+    """
+    prompt = cache_manager.prompts_config_cache.get_config(template_name, version)
+    if "llm_model" in prompt:
+        llm = get_embedding_llm(full_model_name=prompt["llm_model"], temperature=temperature)
+    else:
+        llm = get_embedding_llm(temperature=temperature)
+
+    return prompt["content"], llm
+
+
+def get_transcription_model(model_name: str = "whisper-1") -> TrackedOpenAITranscription:
+    """
+    Get a transcription model instance
+    """
+    api_key = os.getenv('OPENAI_API_KEY')
+    return TrackedOpenAITranscription(
+        api_key=api_key,
+        model=model_name
+    )

common/utils/nginx_utils.py

@@ -6,7 +6,6 @@ def prefixed_url_for(endpoint, **values):
     prefix = request.headers.get('X-Forwarded-Prefix', '')
     scheme = request.headers.get('X-Forwarded-Proto', request.scheme)
     host = request.headers.get('Host', request.host)
-    current_app.logger.debug(f'prefix: {prefix}, scheme: {scheme}, host: {host}')
 
     external = values.pop('_external', False)
     generated_url = url_for(endpoint, **values)

common/utils/os_utils.py

@@ -1,4 +1,6 @@
 import os
+import sys
+
 import gevent
 import time
 from flask import current_app
@@ -28,3 +30,17 @@ def sync_folder(file_path):
     dir_fd = os.open(file_path, os.O_RDONLY)
     os.fsync(dir_fd)
     os.close(dir_fd)
+
+
+def get_project_root():
+    """Get the root directory of the project."""
+    # Use the module that's actually running (not this file)
+    module = sys.modules['__main__']
+    if hasattr(module, '__file__'):
+        # Get the path to the main module
+        main_path = os.path.abspath(module.__file__)
+        # Get the root directory (where the main module is located)
+        return os.path.dirname(main_path)
+    else:
+        # Fallback: use current working directory
+        return os.getcwd()

common/utils/portkey_utils.py

@@ -1,99 +0,0 @@
-import requests
-import json
-from typing import Optional
-
-# Define a function to make the GET request
-def get_metadata_grouped_data(
-    api_key: str,
-    metadata_key: str,
-    time_of_generation_min: Optional[str] = None,
-    time_of_generation_max: Optional[str] = None,
-    total_units_min: Optional[int] = None,
-    total_units_max: Optional[int] = None,
-    cost_min: Optional[float] = None,
-    cost_max: Optional[float] = None,
-    prompt_token_min: Optional[int] = None,
-    prompt_token_max: Optional[int] = None,
-    completion_token_min: Optional[int] = None,
-    completion_token_max: Optional[int] = None,
-    status_code: Optional[str] = None,
-    weighted_feedback_min: Optional[float] = None,
-    weighted_feedback_max: Optional[float] = None,
-    virtual_keys: Optional[str] = None,
-    configs: Optional[str] = None,
-    workspace_slug: Optional[str] = None,
-    api_key_ids: Optional[str] = None,
-    current_page: Optional[int] = 1,
-    page_size: Optional[int] = 20,
-    metadata: Optional[str] = None,
-    ai_org_model: Optional[str] = None,
-    trace_id: Optional[str] = None,
-    span_id: Optional[str] = None,
-):
-    url = f"https://api.portkey.ai/v1/analytics/groups/metadata/{metadata_key}"
-
-    # Set up query parameters
-    params = {
-        "time_of_generation_min": time_of_generation_min,
-        "time_of_generation_max": time_of_generation_max,
-        "total_units_min": total_units_min,
-        "total_units_max": total_units_max,
-        "cost_min": cost_min,
-        "cost_max": cost_max,
-        "prompt_token_min": prompt_token_min,
-        "prompt_token_max": prompt_token_max,
-        "completion_token_min": completion_token_min,
-        "completion_token_max": completion_token_max,
-        "status_code": status_code,
-        "weighted_feedback_min": weighted_feedback_min,
-        "weighted_feedback_max": weighted_feedback_max,
-        "virtual_keys": virtual_keys,
-        "configs": configs,
-        "workspace_slug": workspace_slug,
-        "api_key_ids": api_key_ids,
-        "current_page": current_page,
-        "page_size": page_size,
-        "metadata": metadata,
-        "ai_org_model": ai_org_model,
-        "trace_id": trace_id,
-        "span_id": span_id,
-    }
-
-    # Remove any keys with None values
-    params = {k: v for k, v in params.items() if v is not None}
-
-    # Set up the headers
-    headers = {
-        "Authorization": f"Bearer {api_key}",
-        "Content-Type": "application/json"
-    }
-
-    # Make the GET request
-    response = requests.get(url, headers=headers, params=params)
-
-    # Check for successful response
-    if response.status_code == 200:
-        return response.json()  # Return JSON data
-    else:
-        response.raise_for_status()  # Raise an exception for errors
-
-# Example usage
-# Replace 'your_api_key' and 'your_metadata_key' with actual values
-api_key = 'your_api_key'
-metadata_key = 'your_metadata_key'
-
-try:
-    data = get_metadata_grouped_data(
-        api_key=api_key,
-        metadata_key=metadata_key,
-        time_of_generation_min="2024-08-23T15:50:23+05:30",
-        time_of_generation_max="2024-09-23T15:50:23+05:30",
-        total_units_min=100,
-        total_units_max=1000,
-        cost_min=10,
-        cost_max=100,
-        status_code="200,201"
-    )
-    print(json.dumps(data, indent=4))
-except Exception as e:
-    print(f"Error occurred: {str(e)}")

common/utils/prometheus_utils.py

@@ -0,0 +1,11 @@
+from flask import current_app
+from prometheus_client import push_to_gateway
+
+
+def sanitize_label(value):
+    """Convert value to valid Prometheus label by removing/replacing invalid chars"""
+    if value is None:
+        return ""
+    # Replace spaces and special chars with underscores
+    import re
+    return re.sub(r'[^a-zA-Z0-9_]', '_', str(value))