- Adding Prometheus and grafana services in development

- Adding Prometheus metrics to the business events
- Ensure asynchronous behaviour of crewai specialists.
- Adapt Business events to working in mixed synchronous / asynchronous contexts
- Extend business events with specialist information
- Started adding a grafana dashboard (TBC)

.gitignore

@@ -46,3 +46,9 @@ scripts/__pycache__/run_eveai_app.cpython-312.pyc
 /docker/eveai_logs/
 /integrations/Wordpress/eveai_sync.zip
 /integrations/Wordpress/eveai-chat.zip
+/db_backups/
+/tests/interactive_client/specialist_client.log
+/.repopackignore
+/patched_packages/crewai/
+/docker/prometheus/data/
+/docker/grafana/data/

.idea/misc.xml

@@ -3,5 +3,5 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.12 (eveai_tbd)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (eveai_tbd)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (TBD)" project-jdk-type="Python SDK" />
 </project>

.python-version

@@ -1 +1 @@
-eveai_tbd
+3.12.7

.repopackignore_base

@@ -2,6 +2,7 @@
 # Example:
 # *.log
 # tmp/
+db_backups/
 logs/
 nginx/static/assets/fonts/
 nginx/static/assets/img/
@@ -12,6 +13,7 @@ migrations/
 *material*
 *nucleo*
 *package*
+*.svg
 nginx/mime.types
 *.gitignore*
 .python-version

.repopackignore_eveai_app

@@ -7,5 +7,6 @@ eveai_entitlements/
 eveai_workers/
 instance/
 integrations/
+migrations/
 nginx/
 scripts/

.repopackignore_eveai_app_documents

@@ -0,0 +1,28 @@
+docker/
+eveai_api/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+migrations/
+nginx/
+scripts/
+common/models/entitlements.py
+common/models/interaction.py
+common/models/user.py
+config/agents/
+config/prompts/
+config/specialists/
+config/tasks/
+config/tools/
+eveai_app/templates/administration/
+eveai_app/templates/entitlements/
+eveai_app/templates/interaction/
+eveai_app/templates/user/
+eveai_app/views/administration*
+eveai_app/views/entitlements*
+eveai_app/views/interaction*
+eveai_app/views/user*

.repopackignore_eveai_app_entitlements

@@ -0,0 +1,28 @@
+docker/
+eveai_api/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+migrations/
+nginx/
+scripts/
+common/models/document.py
+common/models/interaction.py
+common/models/user.py
+config/agents/
+config/prompts/
+config/specialists/
+config/tasks/
+config/tools/
+eveai_app/templates/administration/
+eveai_app/templates/document/
+eveai_app/templates/interaction/
+eveai_app/templates/user/
+eveai_app/views/administration*
+eveai_app/views/document*
+eveai_app/views/interaction*
+eveai_app/views/user*

.repopackignore_eveai_app_interaction

@@ -0,0 +1,23 @@
+docker/
+eveai_api/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+migrations/
+nginx/
+scripts/
+common/models/entitlements.py
+common/models/document.py
+common/models/user.py
+eveai_app/templates/administration/
+eveai_app/templates/entitlements/
+eveai_app/templates/document/
+eveai_app/templates/user/
+eveai_app/views/administration*
+eveai_app/views/entitlements*
+eveai_app/views/document*
+eveai_app/views/user*

.repopackignore_eveai_app_startup

@@ -0,0 +1,13 @@
+eveai_api/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+eveai_app/templates/
+eveai_app/views/
+instance/
+integrations/
+migrations/
+nginx/
+scripts/

.repopackignore_eveai_app_user

@@ -0,0 +1,28 @@
+docker/
+eveai_api/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+migrations/
+nginx/
+scripts/
+common/models/entitlements.py
+common/models/interaction.py
+common/models/document.py
+config/agents/
+config/prompts/
+config/specialists/
+config/tasks/
+config/tools/
+eveai_app/templates/administration/
+eveai_app/templates/entitlements/
+eveai_app/templates/interaction/
+eveai_app/templates/document/
+eveai_app/views/administration*
+eveai_app/views/entitlements*
+eveai_app/views/interaction*
+eveai_app/views/document*

.repopackignore_patched_packages

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

CHANGELOG.md

@@ -25,6 +25,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Security
 - In case of vulnerabilities.
 
+## [2.1.0-alfa]
+
+### Added
+- Zapier Refresh Document
+- SPIN Specialist definition - from start to finish
+- Introduction of startup scripts in eveai_app
+- Caching for all configurations added
+- Caching for processed specialist configurations
+- Caching for specialist history
+- Augmented Specialist Editor, including Specialist graphic presentation
+- Introduction of specialist_execution_api, introducting SSE
+- Introduction of crewai framework for specialist implementation
+- Test app for testing specialists - also serves as a sample client application for SSE
+- 
+
+### Changed
+- Improvement of startup of applications using gevent, and better handling and scaling of multiple connections
+- STANDARD_RAG Specialist improvement
+- 
+
+### Deprecated
+- eveai_chat - using sockets - will be replaced with new specialist_execution_api and SSE
+
 ## [2.0.1-alfa]
 
 ### Added

common/eveai_model/eveai_embedding_base.py

@@ -0,0 +1,11 @@
+from abc import abstractmethod
+from typing import List
+
+
+class EveAIEmbeddings:
+    @abstractmethod
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        raise NotImplementedError
+
+    def embed_query(self, text: str) -> List[float]:
+        return self.embed_documents([text])[0]

common/eveai_model/tracked_mistral_embeddings.py

@@ -0,0 +1,40 @@
+from flask import current_app
+from langchain_mistralai import MistralAIEmbeddings
+from typing import List, Any
+import time
+
+from common.eveai_model.eveai_embedding_base import EveAIEmbeddings
+from common.utils.business_event_context import current_event
+from mistralai import Mistral
+
+
+class TrackedMistralAIEmbeddings(EveAIEmbeddings):
+    def __init__(self, model: str = "mistral_embed"):
+        api_key = current_app.config['MISTRAL_API_KEY']
+        self.client = Mistral(
+            api_key=api_key
+        )
+        self.model = model
+        super().__init__()
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        start_time = time.time()
+        result = self.client.embeddings.create(
+            model=self.model,
+            inputs=texts
+        )
+        end_time = time.time()
+
+        metrics = {
+            'total_tokens': result.usage.total_tokens,
+            'prompt_tokens': result.usage.prompt_tokens,  # For embeddings, all tokens are prompt tokens
+            'completion_tokens': result.usage.completion_tokens,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'Embedding',
+        }
+        current_event.log_llm_metrics(metrics)
+
+        embeddings = [embedding.embedding for embedding in result.data]
+
+        return embeddings
+

common/extensions.py

@@ -5,7 +5,6 @@ from flask_security import Security
 from flask_mailman import Mail
 from flask_login import LoginManager
 from flask_cors import CORS
-from flask_socketio import SocketIO
 from flask_jwt_extended import JWTManager
 from flask_session import Session
 from flask_wtf import CSRFProtect
@@ -16,6 +15,7 @@ from .langchain.templates.template_manager import TemplateManager
 from .utils.cache.eveai_cache_manager import EveAICacheManager
 from .utils.simple_encryption import SimpleEncryption
 from .utils.minio_utils import MinioClient
+from .utils.performance_monitoring import EveAIMetrics
 
 
 # Create extensions
@@ -27,7 +27,6 @@ security = Security()
 mail = Mail()
 login_manager = LoginManager()
 cors = CORS()
-socketio = SocketIO()
 jwt = JWTManager()
 session = Session()
 api_rest = Api()
@@ -36,3 +35,5 @@ minio_client = MinioClient()
 metrics = PrometheusMetrics.for_app_factory()
 template_manager = TemplateManager()
 cache_manager = EveAICacheManager()
+eveai_metrics = EveAIMetrics()
+

common/langchain/templates/template_manager.py

@@ -28,7 +28,6 @@ class TemplateManager:
         # Initialize template manager
         base_dir = "/app"
         self.templates_dir = os.path.join(base_dir, 'config', 'prompts')
-        app.logger.debug(f'Loading templates from {self.templates_dir}')
         self.app = app
         self._templates = self._load_templates()
         # Log available templates for each supported model

common/models/document.py

@@ -12,8 +12,8 @@ class Catalog(db.Model):
     description = db.Column(db.Text, nullable=True)
     type = db.Column(db.String(50), nullable=False, default="STANDARD_CATALOG")
 
-    min_chunk_size = db.Column(db.Integer, nullable=True, default=2000)
-    max_chunk_size = db.Column(db.Integer, nullable=True, default=3000)
+    min_chunk_size = db.Column(db.Integer, nullable=True, default=1500)
+    max_chunk_size = db.Column(db.Integer, nullable=True, default=2500)
 
     # Meta Data
     user_metadata = db.Column(JSONB, nullable=True)
@@ -56,6 +56,7 @@ class Retriever(db.Model):
     description = db.Column(db.Text, nullable=True)
     catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True)
     type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="STANDARD_RAG")
     tuning = db.Column(db.Boolean, nullable=True, default=False)
 
     # Meta Data

common/models/entitlements.py

@@ -11,10 +11,13 @@ class BusinessEventLog(db.Model):
     tenant_id = db.Column(db.Integer, nullable=False)
     trace_id = db.Column(db.String(50), nullable=False)
     span_id = db.Column(db.String(50))
-    span_name = db.Column(db.String(50))
+    span_name = db.Column(db.String(255))
     parent_span_id = db.Column(db.String(50))
     document_version_id = db.Column(db.Integer)
     document_version_file_size = db.Column(db.Float)
+    specialist_id = db.Column(db.Integer)
+    specialist_type = db.Column(db.String(50))
+    specialist_type_version = db.Column(db.String(20))
     chat_session_id = db.Column(db.String(50))
     interaction_id = db.Column(db.Integer)
     environment = db.Column(db.String(20))

common/models/interaction.py

@@ -25,6 +25,7 @@ class Specialist(db.Model):
     name = db.Column(db.String(50), nullable=False)
     description = db.Column(db.Text, nullable=True)
     type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
     tuning = db.Column(db.Boolean, nullable=True, default=False)
     configuration = db.Column(JSONB, nullable=True)
     arguments = db.Column(JSONB, nullable=True)
@@ -32,6 +33,141 @@ class Specialist(db.Model):
     # Relationship to retrievers through the association table
     retrievers = db.relationship('SpecialistRetriever', backref='specialist', lazy=True,
                                  cascade="all, delete-orphan")
+    agents = db.relationship('EveAIAgent', backref='specialist', lazy=True)
+    tasks = db.relationship('EveAITask', backref='specialist', lazy=True)
+    tools = db.relationship('EveAITool', backref='specialist', lazy=True)
+    dispatchers = db.relationship('SpecialistDispatcher', backref='specialist', lazy=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class EveAIAsset(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="DOCUMENT_TEMPLATE")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    valid_from = db.Column(db.DateTime, nullable=True)
+    valid_to = db.Column(db.DateTime, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+    # Relations
+    versions = db.relationship('EveAIAssetVersion', backref='asset', lazy=True)
+
+
+class EveAIAssetVersion(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    asset_id = db.Column(db.Integer, db.ForeignKey(EveAIAsset.id), nullable=False)
+    bucket_name = db.Column(db.String(255), nullable=True)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+    # Relations
+    instructions = db.relationship('EveAIAssetInstruction', backref='asset_version', lazy=True)
+
+
+class EveAIAssetInstruction(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    asset_version_id = db.Column(db.Integer, db.ForeignKey(EveAIAssetVersion.id), nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    content = db.Column(db.Text, nullable=True)
+
+
+class EveAIProcessedAsset(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    asset_version_id = db.Column(db.Integer, db.ForeignKey(EveAIAssetVersion.id), nullable=False)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=True)
+    chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=True)
+    bucket_name = db.Column(db.String(255), nullable=True)
+    object_name = db.Column(db.String(255), nullable=True)
+    created_at = db.Column(db.DateTime, nullable=True, server_default=db.func.now())
+
+
+class EveAIAgent(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    role = db.Column(db.Text, nullable=True)
+    goal = db.Column(db.Text, nullable=True)
+    backstory = db.Column(db.Text, nullable=True)
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class EveAITask(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    task_description = db.Column(db.Text, nullable=True)
+    expected_output = db.Column(db.Text, nullable=True)
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+    context = db.Column(JSONB, nullable=True)
+    asynchronous = db.Column(db.Boolean, nullable=True, default=False)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class EveAITool(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=False)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
+
+
+class Dispatcher(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(50), nullable=False, default="STANDARD_RAG")
+    type_version = db.Column(db.String(20), nullable=True, default="1.0.0")
+    tuning = db.Column(db.Boolean, nullable=True, default=False)
+    configuration = db.Column(JSONB, nullable=True)
+    arguments = db.Column(JSONB, nullable=True)
 
     # Versioning Information
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
@@ -71,3 +207,10 @@ class SpecialistRetriever(db.Model):
     retriever_id = db.Column(db.Integer, db.ForeignKey(Retriever.id, ondelete='CASCADE'), primary_key=True)
 
     retriever = db.relationship("Retriever", backref="specialist_retrievers")
+
+
+class SpecialistDispatcher(db.Model):
+    specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True)
+    dispatcher_id = db.Column(db.Integer, db.ForeignKey(Dispatcher.id, ondelete='CASCADE'), primary_key=True)
+
+    dispatcher = db.relationship("Dispatcher", backref="specialist_dispatchers")

common/models/user.py

@@ -31,7 +31,6 @@ class Tenant(db.Model):
     allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True)
 
     # LLM specific choices
-    embedding_model = db.Column(db.String(50), nullable=True)
     llm_model = db.Column(db.String(50), nullable=True)
 
     # Entitlements
@@ -66,7 +65,6 @@ class Tenant(db.Model):
             'type': self.type,
             'default_language': self.default_language,
             'allowed_languages': self.allowed_languages,
-            'embedding_model': self.embedding_model,
             'llm_model': self.llm_model,
             'currency': self.currency,
         }

common/utils/asset_utils.py

@@ -0,0 +1,62 @@
+from datetime import datetime as dt, timezone as tz
+
+from flask import current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import cache_manager, minio_client, db
+from common.models.interaction import EveAIAsset, EveAIAssetVersion
+from common.utils.document_utils import mark_tenant_storage_dirty
+from common.utils.model_logging_utils import set_logging_information
+
+
+def create_asset_stack(api_input, tenant_id):
+    type_version = cache_manager.assets_version_tree_cache.get_latest_version(api_input['type'])
+    api_input['type_version'] = type_version
+    new_asset = create_asset(api_input, tenant_id)
+    new_asset_version = create_version_for_asset(new_asset, tenant_id)
+    db.session.add(new_asset)
+    db.session.add(new_asset_version)
+
+    try:
+        db.session.commit()
+    except SQLAlchemyError as e:
+        current_app.logger.error(f"Could not add asset for tenant {tenant_id}: {str(e)}")
+        db.session.rollback()
+        raise e
+
+    return new_asset, new_asset_version
+
+
+def create_asset(api_input, tenant_id):
+    new_asset = EveAIAsset()
+    new_asset.name = api_input['name']
+    new_asset.description = api_input['description']
+    new_asset.type = api_input['type']
+    new_asset.type_version = api_input['type_version']
+    if api_input['valid_from'] and api_input['valid_from'] != '':
+        new_asset.valid_from = api_input['valid_from']
+    else:
+        new_asset.valid_from = dt.now(tz.utc)
+    new_asset.valid_to = api_input['valid_to']
+    set_logging_information(new_asset, dt.now(tz.utc))
+
+    return new_asset
+
+
+def create_version_for_asset(asset, tenant_id):
+    new_asset_version = EveAIAssetVersion()
+    new_asset_version.asset = asset
+    new_asset_version.bucket_name = minio_client.create_tenant_bucket(tenant_id)
+    set_logging_information(new_asset_version, dt.now(tz.utc))
+
+    return new_asset_version
+
+
+def add_asset_version_file(asset_version, field_name, file, tenant_id):
+    object_name, file_size = minio_client.upload_file(asset_version.bucket_name, asset_version.id, field_name,
+                                                      file.content_type)
+    mark_tenant_storage_dirty(tenant_id)
+    return object_name
+
+
+

common/utils/business_event.py

@@ -1,14 +1,81 @@
 import os
+import time
 import uuid
-from contextlib import contextmanager
+from contextlib import contextmanager, asynccontextmanager
 from datetime import datetime
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 from datetime import datetime as dt, timezone as tz
 import logging
+from prometheus_client import Counter, Histogram, Gauge, Summary
 
 from .business_event_context import BusinessEventContext
 from common.models.entitlements import BusinessEventLog
 from common.extensions import db
+from .celery_utils import current_celery
+from common.utils.performance_monitoring import EveAIMetrics
+
+# Standard duration buckets for all histograms
+DURATION_BUCKETS = EveAIMetrics.get_standard_buckets()
+
+# Prometheus metrics for business events
+TRACE_COUNTER = Counter(
+    'eveai_business_events_total',
+    'Total number of business events triggered',
+    ['tenant_id', 'event_type', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+TRACE_DURATION = Histogram(
+    'eveai_business_events_duration_seconds',
+    'Duration of business events in seconds',
+    ['tenant_id', 'event_type', 'specialist_id', 'specialist_type', 'specialist_type_version'],
+    buckets=DURATION_BUCKETS
+)
+
+CONCURRENT_TRACES = Gauge(
+    'eveai_business_events_concurrent',
+    'Number of concurrent business events',
+    ['tenant_id', 'event_type', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+SPAN_COUNTER = Counter(
+    'eveai_business_spans_total',
+    'Total number of spans within business events',
+    ['tenant_id', 'event_type', 'activity_name', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+SPAN_DURATION = Histogram(
+    'eveai_business_spans_duration_seconds',
+    'Duration of spans within business events in seconds',
+    ['tenant_id', 'event_type', 'activity_name', 'specialist_id', 'specialist_type', 'specialist_type_version'],
+    buckets=DURATION_BUCKETS
+)
+
+CONCURRENT_SPANS = Gauge(
+    'eveai_business_spans_concurrent',
+    'Number of concurrent spans within business events',
+    ['tenant_id', 'event_type', 'activity_name', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
+
+# LLM Usage metrics
+LLM_TOKENS_COUNTER = Counter(
+    'eveai_llm_tokens_total',
+    'Total number of tokens used in LLM calls',
+    ['tenant_id', 'event_type', 'interaction_type', 'token_type', 'specialist_id', 'specialist_type',
+     'specialist_type_version']
+)
+
+LLM_DURATION = Histogram(
+    'eveai_llm_duration_seconds',
+    'Duration of LLM API calls in seconds',
+    ['tenant_id', 'event_type', 'interaction_type', 'specialist_id', 'specialist_type', 'specialist_type_version'],
+    buckets=DURATION_BUCKETS
+)
+
+LLM_CALLS_COUNTER = Counter(
+    'eveai_llm_calls_total',
+    'Total number of LLM API calls',
+    ['tenant_id', 'event_type', 'interaction_type', 'specialist_id', 'specialist_type', 'specialist_type_version']
+)
 
 
 class BusinessEvent:
@@ -27,6 +94,9 @@ class BusinessEvent:
         self.document_version_file_size = kwargs.get('document_version_file_size')
         self.chat_session_id = kwargs.get('chat_session_id')
         self.interaction_id = kwargs.get('interaction_id')
+        self.specialist_id = kwargs.get('specialist_id')
+        self.specialist_type = kwargs.get('specialist_type')
+        self.specialist_type_version = kwargs.get('specialist_type_version')
         self.environment = os.environ.get("FLASK_ENV", "development")
         self.span_counter = 0
         self.spans = []
@@ -38,10 +108,44 @@ class BusinessEvent:
             'call_count': 0,
             'interaction_type': None
         }
+        self._log_buffer = []
+
+        # Prometheus label values must be strings
+        self.tenant_id_str = str(self.tenant_id)
+        self.specialist_id_str = str(self.specialist_id) if self.specialist_id else ""
+        self.specialist_type_str = str(self.specialist_type) if self.specialist_type else ""
+        self.specialist_type_version_str = str(self.specialist_type_version) if self.specialist_type_version else ""
+
+        # Increment concurrent events gauge when initialized
+        CONCURRENT_TRACES.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        # Increment trace counter
+        TRACE_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
 
     def update_attribute(self, attribute: str, value: any):
         if hasattr(self, attribute):
             setattr(self, attribute, value)
+            # Update string versions for Prometheus labels if needed
+            if attribute == 'specialist_id':
+                self.specialist_id_str = str(value) if value else ""
+            elif attribute == 'specialist_type':
+                self.specialist_type_str = str(value) if value else ""
+            elif attribute == 'specialist_type_version':
+                self.specialist_type_version_str = str(value) if value else ""
+            elif attribute == 'tenant_id':
+                self.tenant_id_str = str(value)
         else:
             raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'")
 
@@ -53,6 +157,60 @@ class BusinessEvent:
         self.llm_metrics['call_count'] += 1
         self.llm_metrics['interaction_type'] = metrics['interaction_type']
 
+        # Track in Prometheus metrics
+        interaction_type = metrics['interaction_type']
+
+        # Track token usage
+        LLM_TOKENS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            interaction_type=interaction_type,
+            token_type='total',
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc(metrics['total_tokens'])
+
+        LLM_TOKENS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            interaction_type=interaction_type,
+            token_type='prompt',
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc(metrics['prompt_tokens'])
+
+        LLM_TOKENS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            interaction_type=interaction_type,
+            token_type='completion',
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc(metrics['completion_tokens'])
+
+        # Track duration
+        LLM_DURATION.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            interaction_type=interaction_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).observe(metrics['time_elapsed'])
+
+        # Track call count
+        LLM_CALLS_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            interaction_type=interaction_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
     def reset_llm_metrics(self):
         self.llm_metrics['total_tokens'] = 0
         self.llm_metrics['prompt_tokens'] = 0
@@ -80,15 +238,61 @@ class BusinessEvent:
         self.span_name = span_name
         self.parent_span_id = parent_span_id
 
-        self.log(f"Starting span {span_name}")
+        # Track start time for the span
+        span_start_time = time.time()
+
+        # Increment span metrics - using span_name as activity_name for metrics
+        SPAN_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            activity_name=span_name,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        # Increment concurrent spans gauge
+        CONCURRENT_SPANS.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            activity_name=span_name,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        self.log(f"Start")
 
         try:
             yield
         finally:
+            # Calculate total time for this span
+            span_total_time = time.time() - span_start_time
+
+            # Observe span duration
+            SPAN_DURATION.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type,
+                activity_name=span_name,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).observe(span_total_time)
+
+            # Decrement concurrent spans gauge
+            CONCURRENT_SPANS.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type,
+                activity_name=span_name,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).dec()
+
             if self.llm_metrics['call_count'] > 0:
                 self.log_final_metrics()
                 self.reset_llm_metrics()
-            self.log(f"Ending span {span_name}")
+            self.log(f"End", extra_fields={'span_duration': span_total_time})
             # Restore the previous span info
             if self.spans:
                 self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
@@ -97,9 +301,87 @@ class BusinessEvent:
                 self.span_name = None
                 self.parent_span_id = None
 
-    def log(self, message: str, level: str = 'info'):
-        logger = logging.getLogger('business_events')
+    @asynccontextmanager
+    async def create_span_async(self, span_name: str):
+        """Async version of create_span using async context manager"""
+        parent_span_id = self.span_id
+        self.span_counter += 1
+        new_span_id = str(uuid.uuid4())
+
+        # Save the current span info
+        self.spans.append((self.span_id, self.span_name, self.parent_span_id))
+
+        # Set the new span info
+        self.span_id = new_span_id
+        self.span_name = span_name
+        self.parent_span_id = parent_span_id
+
+        # Track start time for the span
+        span_start_time = time.time()
+
+        # Increment span metrics - using span_name as activity_name for metrics
+        SPAN_COUNTER.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            activity_name=span_name,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        # Increment concurrent spans gauge
+        CONCURRENT_SPANS.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            activity_name=span_name,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).inc()
+
+        self.log(f"Start")
+
+        try:
+            yield
+        finally:
+            # Calculate total time for this span
+            span_total_time = time.time() - span_start_time
+
+            # Observe span duration
+            SPAN_DURATION.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type,
+                activity_name=span_name,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).observe(span_total_time)
+
+            # Decrement concurrent spans gauge
+            CONCURRENT_SPANS.labels(
+                tenant_id=self.tenant_id_str,
+                event_type=self.event_type,
+                activity_name=span_name,
+                specialist_id=self.specialist_id_str,
+                specialist_type=self.specialist_type_str,
+                specialist_type_version=self.specialist_type_version_str
+            ).dec()
+
+            if self.llm_metrics['call_count'] > 0:
+                self.log_final_metrics()
+                self.reset_llm_metrics()
+            self.log(f"End", extra_fields={'span_duration': span_total_time})
+            # Restore the previous span info
+            if self.spans:
+                self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+            else:
+                self.span_id = None
+                self.span_name = None
+                self.parent_span_id = None
+
+    def log(self, message: str, level: str = 'info', extra_fields: Dict[str, Any] = None):
         log_data = {
+            'timestamp': dt.now(tz=tz.utc),
             'event_type': self.event_type,
             'tenant_id': self.tenant_id,
             'trace_id': self.trace_id,
@@ -110,35 +392,29 @@ class BusinessEvent:
             'document_version_file_size': self.document_version_file_size,
             'chat_session_id': self.chat_session_id,
             'interaction_id': self.interaction_id,
+            'specialist_id': self.specialist_id,
+            'specialist_type': self.specialist_type,
+            'specialist_type_version': self.specialist_type_version,
             'environment': self.environment,
+            'message': message,
         }
-        # log to Graylog
-        getattr(logger, level)(message, extra=log_data)
+        # Add any extra fields
+        if extra_fields:
+            for key, value in extra_fields.items():
+                # For span/trace duration, use the llm_metrics_total_time field
+                if key == 'span_duration' or key == 'trace_duration':
+                    log_data['llm_metrics_total_time'] = value
+                else:
+                    log_data[key] = value
 
-        # Log to database
-        event_log = BusinessEventLog(
-            timestamp=dt.now(tz=tz.utc),
-            event_type=self.event_type,
-            tenant_id=self.tenant_id,
-            trace_id=self.trace_id,
-            span_id=self.span_id,
-            span_name=self.span_name,
-            parent_span_id=self.parent_span_id,
-            document_version_id=self.document_version_id,
-            document_version_file_size=self.document_version_file_size,
-            chat_session_id=self.chat_session_id,
-            interaction_id=self.interaction_id,
-            environment=self.environment,
-            message=message
-        )
-        db.session.add(event_log)
-        db.session.commit()
+        self._log_buffer.append(log_data)
 
     def log_llm_metrics(self, metrics: dict, level: str = 'info'):
         self.update_llm_metrics(metrics)
         message = "LLM Metrics"
         logger = logging.getLogger('business_events')
         log_data = {
+            'timestamp': dt.now(tz=tz.utc),
             'event_type': self.event_type,
             'tenant_id': self.tenant_id,
             'trace_id': self.trace_id,
@@ -149,44 +425,24 @@ class BusinessEvent:
             'document_version_file_size': self.document_version_file_size,
             'chat_session_id': self.chat_session_id,
             'interaction_id': self.interaction_id,
+            'specialist_id': self.specialist_id,
+            'specialist_type': self.specialist_type,
+            'specialist_type_version': self.specialist_type_version,
             'environment': self.environment,
             'llm_metrics_total_tokens': metrics['total_tokens'],
             'llm_metrics_prompt_tokens': metrics['prompt_tokens'],
             'llm_metrics_completion_tokens': metrics['completion_tokens'],
             'llm_metrics_total_time': metrics['time_elapsed'],
             'llm_interaction_type': metrics['interaction_type'],
+            'message': message,
         }
-        # log to Graylog
-        getattr(logger, level)(message, extra=log_data)
-
-        # Log to database
-        event_log = BusinessEventLog(
-            timestamp=dt.now(tz=tz.utc),
-            event_type=self.event_type,
-            tenant_id=self.tenant_id,
-            trace_id=self.trace_id,
-            span_id=self.span_id,
-            span_name=self.span_name,
-            parent_span_id=self.parent_span_id,
-            document_version_id=self.document_version_id,
-            document_version_file_size=self.document_version_file_size,
-            chat_session_id=self.chat_session_id,
-            interaction_id=self.interaction_id,
-            environment=self.environment,
-            llm_metrics_total_tokens=metrics['total_tokens'],
-            llm_metrics_prompt_tokens=metrics['prompt_tokens'],
-            llm_metrics_completion_tokens=metrics['completion_tokens'],
-            llm_metrics_total_time=metrics['time_elapsed'],
-            llm_interaction_type=metrics['interaction_type'],
-            message=message
-        )
-        db.session.add(event_log)
-        db.session.commit()
+        self._log_buffer.append(log_data)
 
     def log_final_metrics(self, level: str = 'info'):
         logger = logging.getLogger('business_events')
         message = "Final LLM Metrics"
         log_data = {
+            'timestamp': dt.now(tz=tz.utc),
             'event_type': self.event_type,
             'tenant_id': self.tenant_id,
             'trace_id': self.trace_id,
@@ -197,6 +453,9 @@ class BusinessEvent:
             'document_version_file_size': self.document_version_file_size,
             'chat_session_id': self.chat_session_id,
             'interaction_id': self.interaction_id,
+            'specialist_id': self.specialist_id,
+            'specialist_type': self.specialist_type,
+            'specialist_type_version': self.specialist_type_version,
             'environment': self.environment,
             'llm_metrics_total_tokens': self.llm_metrics['total_tokens'],
             'llm_metrics_prompt_tokens': self.llm_metrics['prompt_tokens'],
@@ -204,42 +463,133 @@ class BusinessEvent:
             'llm_metrics_total_time': self.llm_metrics['total_time'],
             'llm_metrics_call_count': self.llm_metrics['call_count'],
             'llm_interaction_type': self.llm_metrics['interaction_type'],
+            'message': message,
         }
-        # log to Graylog
-        getattr(logger, level)(message, extra=log_data)
+        self._log_buffer.append(log_data)
 
-        # Log to database
-        event_log = BusinessEventLog(
-            timestamp=dt.now(tz=tz.utc),
-            event_type=self.event_type,
-            tenant_id=self.tenant_id,
-            trace_id=self.trace_id,
-            span_id=self.span_id,
-            span_name=self.span_name,
-            parent_span_id=self.parent_span_id,
-            document_version_id=self.document_version_id,
-            document_version_file_size=self.document_version_file_size,
-            chat_session_id=self.chat_session_id,
-            interaction_id=self.interaction_id,
-            environment=self.environment,
-            llm_metrics_total_tokens=self.llm_metrics['total_tokens'],
-            llm_metrics_prompt_tokens=self.llm_metrics['prompt_tokens'],
-            llm_metrics_completion_tokens=self.llm_metrics['completion_tokens'],
-            llm_metrics_total_time=self.llm_metrics['total_time'],
-            llm_metrics_call_count=self.llm_metrics['call_count'],
-            llm_interaction_type=self.llm_metrics['interaction_type'],
-            message=message
-        )
-        db.session.add(event_log)
-        db.session.commit()
+    @staticmethod
+    def _direct_db_persist(log_entries: List[Dict[str, Any]]):
+        """Fallback method to directly persist logs to DB if async fails"""
+        try:
+            db_entries = []
+            for entry in log_entries:
+                event_log = BusinessEventLog(
+                    timestamp=entry.pop('timestamp'),
+                    event_type=entry.pop('event_type'),
+                    tenant_id=entry.pop('tenant_id'),
+                    trace_id=entry.pop('trace_id'),
+                    span_id=entry.pop('span_id', None),
+                    span_name=entry.pop('span_name', None),
+                    parent_span_id=entry.pop('parent_span_id', None),
+                    document_version_id=entry.pop('document_version_id', None),
+                    document_version_file_size=entry.pop('document_version_file_size', None),
+                    chat_session_id=entry.pop('chat_session_id', None),
+                    interaction_id=entry.pop('interaction_id', None),
+                    specialist_id=entry.pop('specialist_id', None),
+                    specialist_type=entry.pop('specialist_type', None),
+                    specialist_type_version=entry.pop('specialist_type_version', None),
+                    environment=entry.pop('environment', None),
+                    llm_metrics_total_tokens=entry.pop('llm_metrics_total_tokens', None),
+                    llm_metrics_prompt_tokens=entry.pop('llm_metrics_prompt_tokens', None),
+                    llm_metrics_completion_tokens=entry.pop('llm_metrics_completion_tokens', None),
+                    llm_metrics_total_time=entry.pop('llm_metrics_total_time', None),
+                    llm_metrics_call_count=entry.pop('llm_metrics_call_count', None),
+                    llm_interaction_type=entry.pop('llm_interaction_type', None),
+                    message=entry.pop('message', None)
+                )
+                db_entries.append(event_log)
+
+            # Bulk insert
+            db.session.bulk_save_objects(db_entries)
+            db.session.commit()
+        except Exception as e:
+            logger = logging.getLogger('business_events')
+            logger.error(f"Failed to persist logs directly to DB: {e}")
+            db.session.rollback()
+
+    def _flush_log_buffer(self):
+        """Flush the log buffer to the database via a Celery task"""
+        if self._log_buffer:
+            try:
+                # Send to Celery task
+                current_celery.send_task(
+                    'persist_business_events',
+                    args=[self._log_buffer],
+                    queue='entitlements'  # Or dedicated log queue
+                )
+            except Exception as e:
+                # Fallback to direct DB write in case of issues with Celery
+                logger = logging.getLogger('business_events')
+                logger.error(f"Failed to send logs to Celery. Falling back to direct DB: {e}")
+                self._direct_db_persist(self._log_buffer)
+
+            # Clear the buffer after sending
+            self._log_buffer = []
 
     def __enter__(self):
+        self.trace_start_time = time.time()
         self.log(f'Starting Trace for {self.event_type}')
         return BusinessEventContext(self).__enter__()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        trace_total_time = time.time() - self.trace_start_time
+
+        # Record trace duration
+        TRACE_DURATION.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).observe(trace_total_time)
+
+        # Decrement concurrent traces gauge
+        CONCURRENT_TRACES.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).dec()
+
         if self.llm_metrics['call_count'] > 0:
             self.log_final_metrics()
             self.reset_llm_metrics()
-        self.log(f'Ending Trace for {self.event_type}')
+
+        self.log(f'Ending Trace for {self.event_type}', extra_fields={'trace_duration': trace_total_time})
+        self._flush_log_buffer()
         return BusinessEventContext(self).__exit__(exc_type, exc_val, exc_tb)
+
+    async def __aenter__(self):
+        self.trace_start_time = time.time()
+        self.log(f'Starting Trace for {self.event_type}')
+        return await BusinessEventContext(self).__aenter__()
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        trace_total_time = time.time() - self.trace_start_time
+
+        # Record trace duration
+        TRACE_DURATION.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).observe(trace_total_time)
+
+        # Decrement concurrent traces gauge
+        CONCURRENT_TRACES.labels(
+            tenant_id=self.tenant_id_str,
+            event_type=self.event_type,
+            specialist_id=self.specialist_id_str,
+            specialist_type=self.specialist_type_str,
+            specialist_type_version=self.specialist_type_version_str
+        ).dec()
+
+        if self.llm_metrics['call_count'] > 0:
+            self.log_final_metrics()
+            self.reset_llm_metrics()
+
+        self.log(f'Ending Trace for {self.event_type}', extra_fields={'trace_duration': trace_total_time})
+        self._flush_log_buffer()
+        return await BusinessEventContext(self).__aexit__(exc_type, exc_val, exc_tb)

common/utils/business_event_context.py

@@ -1,9 +1,22 @@
 from werkzeug.local import LocalProxy, LocalStack
+import asyncio
+from contextvars import ContextVar
+import contextvars
 
+# Keep existing stack for backward compatibility
 _business_event_stack = LocalStack()
 
+# Add contextvar for async support
+_business_event_contextvar = ContextVar('business_event', default=None)
+
 
 def _get_current_event():
+    # Try contextvar first (for async)
+    event = _business_event_contextvar.get()
+    if event is not None:
+        return event
+
+    # Fall back to the stack-based approach (for sync)
     top = _business_event_stack.top
     if top is None:
         raise RuntimeError("No business event context found. Are you sure you're in a business event?")
@@ -16,10 +29,24 @@ current_event = LocalProxy(_get_current_event)
 class BusinessEventContext:
     def __init__(self, event):
         self.event = event
+        self._token = None  # For storing contextvar token
 
     def __enter__(self):
         _business_event_stack.push(self.event)
+        self._token = _business_event_contextvar.set(self.event)
         return self.event
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         _business_event_stack.pop()
+        if self._token is not None:
+            _business_event_contextvar.reset(self._token)
+
+    async def __aenter__(self):
+        _business_event_stack.push(self.event)
+        self._token = _business_event_contextvar.set(self.event)
+        return self.event
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        _business_event_stack.pop()
+        if self._token is not None:
+            _business_event_contextvar.reset(self._token)

common/utils/cache/base.py

@@ -1,89 +1,192 @@
-# common/utils/cache/base.py
-
 from typing import Any, Dict, List, Optional, TypeVar, Generic, Type
 from dataclasses import dataclass
-from flask import Flask
+from flask import Flask, current_app
 from dogpile.cache import CacheRegion
+from abc import ABC, abstractmethod
 
-T = TypeVar('T')
+T = TypeVar('T')  # Generic type parameter for cached data
 
 
 @dataclass
 class CacheKey:
-    """Represents a cache key with multiple components"""
+    """
+    Represents a composite cache key made up of multiple components.
+    Enables structured and consistent key generation for cache entries.
+
+    Attributes:
+        components (Dict[str, Any]): Dictionary of key components and their values
+
+    Example:
+        key = CacheKey({'tenant_id': 123, 'user_id': 456})
+        str(key) -> "tenant_id=123:user_id=456"
+    """
     components: Dict[str, Any]
 
     def __str__(self) -> str:
+        """
+        Converts components into a deterministic string representation.
+        Components are sorted alphabetically to ensure consistent key generation.
+        """
         return ":".join(f"{k}={v}" for k, v in sorted(self.components.items()))
 
 
-class CacheInvalidationManager:
-    """Manages cache invalidation subscriptions"""
-
-    def __init__(self):
-        self._subscribers = {}
-
-    def subscribe(self, model: str, handler: 'CacheHandler', key_fields: List[str]):
-        if model not in self._subscribers:
-            self._subscribers[model] = []
-        self._subscribers[model].append((handler, key_fields))
-
-    def notify_change(self, model: str, **identifiers):
-        if model in self._subscribers:
-            for handler, key_fields in self._subscribers[model]:
-                if all(field in identifiers for field in key_fields):
-                    handler.invalidate_by_model(model, **identifiers)
-
-
 class CacheHandler(Generic[T]):
-    """Base cache handler implementation"""
+    """
+    Base cache handler implementation providing structured caching functionality.
+    Uses generics to ensure type safety of cached data.
+
+    Type Parameters:
+        T: Type of data being cached
+
+    Attributes:
+        region (CacheRegion): Dogpile cache region for storage
+        prefix (str): Prefix for all cache keys managed by this handler
+    """
 
     def __init__(self, region: CacheRegion, prefix: str):
         self.region = region
         self.prefix = prefix
-        self._key_components = []
+        self._key_components = []  # List of required key components
+
+    @abstractmethod
+    def _to_cache_data(self, instance: T) -> Any:
+        """
+        Convert the data to a cacheable format for internal use.
+
+        Args:
+            instance: The data to be cached.
+
+        Returns:
+            A serializable format of the instance.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _from_cache_data(self, data: Any, **kwargs) -> T:
+        """
+        Convert cached data back to usable format for internal use.
+
+        Args:
+            data: The cached data.
+            **kwargs: Additional context.
+
+        Returns:
+            The data in its usable format.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _should_cache(self, value: T) -> bool:
+        """
+        Validate if the value should be cached for internal use.
+
+        Args:
+            value: The value to be cached.
+
+        Returns:
+            True if the value should be cached, False otherwise.
+        """
+        raise NotImplementedError
 
     def configure_keys(self, *components: str):
+        """
+        Configure required components for cache key generation.
+
+        Args:
+            *components: Required key component names
+
+        Returns:
+            self for method chaining
+        """
         self._key_components = components
         return self
 
-    def subscribe_to_model(self, model: str, key_fields: List[str]):
-        invalidation_manager.subscribe(model, self, key_fields)
-        return self
-
     def generate_key(self, **identifiers) -> str:
+        """
+        Generate a cache key from provided identifiers.
+
+        Args:
+            **identifiers: Key-value pairs for key components
+
+        Returns:
+            Formatted cache key string
+
+        Raises:
+            ValueError: If required components are missing
+        """
         missing = set(self._key_components) - set(identifiers.keys())
         if missing:
             raise ValueError(f"Missing key components: {missing}")
 
+        region_name = getattr(self.region, 'name', 'default_region')
+
         key = CacheKey({k: identifiers[k] for k in self._key_components})
-        return f"{self.prefix}:{str(key)}"
+        return f"{region_name}_{self.prefix}:{str(key)}"
 
     def get(self, creator_func, **identifiers) -> T:
+        """
+        Get or create a cached value.
+
+        Args:
+            creator_func: Function to create value if not cached
+            **identifiers: Key components for cache key
+
+        Returns:
+            Cached or newly created value
+        """
         cache_key = self.generate_key(**identifiers)
 
         def creator():
             instance = creator_func(**identifiers)
-            return self.to_cache_data(instance)
+            serialized_instance = self._to_cache_data(instance)
+            return serialized_instance
 
         cached_data = self.region.get_or_create(
             cache_key,
             creator,
-            should_cache_fn=self.should_cache
+            should_cache_fn=self._should_cache
         )
 
-        return self.from_cache_data(cached_data, **identifiers)
+        return self._from_cache_data(cached_data, **identifiers)
 
     def invalidate(self, **identifiers):
+        """
+        Invalidate a specific cache entry.
+
+        Args:
+            **identifiers: Key components for the cache entry
+        """
         cache_key = self.generate_key(**identifiers)
         self.region.delete(cache_key)
 
     def invalidate_by_model(self, model: str, **identifiers):
+        """
+        Invalidate cache entry based on model changes.
+
+        Args:
+            model: Changed model name
+            **identifiers: Model instance identifiers
+        """
         try:
             self.invalidate(**identifiers)
         except ValueError:
-            pass
+            pass  # Skip if cache key can't be generated from provided identifiers
 
+    def invalidate_region(self):
+        """
+        Invalidate all cache entries within this region.
 
-# Create global invalidation manager
-invalidation_manager = CacheInvalidationManager()
+        Deletes all keys that start with the region prefix.
+        """
+        # Construct the pattern for all keys in this region
+        pattern = f"{self.region}_{self.prefix}:*"
+
+        # Assuming Redis backend with dogpile, use `delete_multi` or direct Redis access
+        if hasattr(self.region.backend, 'client'):
+            redis_client = self.region.backend.client
+            keys_to_delete = redis_client.keys(pattern)
+            if keys_to_delete:
+                redis_client.delete(*keys_to_delete)
+        else:
+            # Fallback for other backends
+            raise NotImplementedError("Region invalidation is only supported for Redis backend.")

common/utils/cache/config_cache.py

@@ -0,0 +1,468 @@
+from typing import Dict, Any, Optional
+from pathlib import Path
+import yaml
+from packaging import version
+import os
+from flask import current_app
+
+from common.utils.cache.base import CacheHandler, CacheKey
+from config.type_defs import agent_types, task_types, tool_types, specialist_types, retriever_types, prompt_types, \
+    catalog_types
+
+
+def is_major_minor(version: str) -> bool:
+    parts = version.strip('.').split('.')
+    return len(parts) == 2 and all(part.isdigit() for part in parts)
+
+
+class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
+    """Base handler for configuration caching"""
+
+    def __init__(self, region, config_type: str):
+        """
+        Args:
+            region: Cache region
+            config_type: Type of configuration (agents, tasks, etc.)
+        """
+        super().__init__(region, f'config_{config_type}')
+        self.config_type = config_type
+        self._types_module = None  # Set by subclasses
+        self._config_dir = None  # Set by subclasses
+        self.version_tree_cache = None
+        self.configure_keys('type_name', 'version')
+
+    def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert the data to a cacheable format"""
+        # For configuration data, we can just return the dictionary as is
+        # since it's already in a serializable format
+        return instance
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Convert cached data back to usable format"""
+        # Similarly, we can return the data directly since it's already
+        # in the format we need
+        return data
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """
+        Validate if the value should be cached
+
+        Args:
+            value: The value to be cached
+
+        Returns:
+            bool: True if the value should be cached
+        """
+        return isinstance(value, dict)  # Cache all dictionaries
+
+    def set_version_tree_cache(self, cache):
+        """Set the version tree cache dependency."""
+        self.version_tree_cache = cache
+
+    def _load_specific_config(self, type_name: str, version_str: str) -> Dict[str, Any]:
+        """
+        Load a specific configuration version
+
+        Args:
+            type_name: Type name
+            version_str: Version string
+
+        Returns:
+            Configuration data
+        """
+        version_tree = self.version_tree_cache.get_versions(type_name)
+        versions = version_tree['versions']
+
+        if version_str == 'latest':
+            version_str = version_tree['latest_version']
+
+        if version_str not in versions:
+            raise ValueError(f"Version {version_str} not found for {type_name}")
+
+        file_path = versions[version_str]['file_path']
+
+        try:
+            with open(file_path) as f:
+                config = yaml.safe_load(f)
+                return config
+        except Exception as e:
+            raise ValueError(f"Error loading config from {file_path}: {e}")
+
+    def get_config(self, type_name: str, version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get configuration for a specific type and version
+        If version not specified, returns latest
+
+        Args:
+            type_name: Configuration type name
+            version: Optional specific version to retrieve
+
+        Returns:
+            Configuration data
+        """
+        if version is None:
+            version_str = self.version_tree_cache.get_latest_version(type_name)
+        elif is_major_minor(version):
+            version_str = self.version_tree_cache.get_latest_patch_version(type_name, version)
+        else:
+            version_str = version
+
+        result = self.get(
+            lambda type_name, version: self._load_specific_config(type_name, version),
+            type_name=type_name,
+            version=version_str
+        )
+        return result
+
+
+class BaseConfigVersionTreeCacheHandler(CacheHandler[Dict[str, Any]]):
+    """Base handler for configuration version tree caching"""
+
+    def __init__(self, region, config_type: str):
+        """
+        Args:
+            region: Cache region
+            config_type: Type of configuration (agents, tasks, etc.)
+        """
+        super().__init__(region, f'config_{config_type}_version_tree')
+        self.config_type = config_type
+        self._types_module = None  # Set by subclasses
+        self._config_dir = None  # Set by subclasses
+        self.configure_keys('type_name')
+
+    def _load_version_tree(self, type_name: str) -> Dict[str, Any]:
+        """
+        Load version tree for a specific type without loading full configurations
+
+        Args:
+            type_name: Name of configuration type
+
+        Returns:
+            Dict containing available versions and their metadata
+        """
+        type_path = Path(self._config_dir) / type_name
+        if not type_path.exists():
+            raise ValueError(f"No configuration found for type {type_name}")
+
+        version_files = list(type_path.glob('*.yaml'))
+        if not version_files:
+            raise ValueError(f"No versions found for type {type_name}")
+
+        versions = {}
+        latest_version = None
+        latest_version_obj = None
+
+        for file_path in version_files:
+            ver = file_path.stem  # Get version from filename
+            try:
+                ver_obj = version.parse(ver)
+                # Only load minimal metadata for version tree
+                with open(file_path) as f:
+                    yaml_data = yaml.safe_load(f)
+                    metadata = yaml_data.get('metadata', {})
+                    versions[ver] = {
+                        'metadata': metadata,
+                        'file_path': str(file_path)
+                    }
+
+                # Track latest version
+                if latest_version_obj is None or ver_obj > latest_version_obj:
+                    latest_version = ver
+                    latest_version_obj = ver_obj
+
+            except Exception as e:
+                current_app.logger.error(f"Error loading version {ver}: {e}")
+                continue
+
+        return {
+            'versions': versions,
+            'latest_version': latest_version
+        }
+
+    def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert the data to a cacheable format"""
+        # For configuration data, we can just return the dictionary as is
+        # since it's already in a serializable format
+        return instance
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Convert cached data back to usable format"""
+        # Similarly, we can return the data directly since it's already
+        # in the format we need
+        return data
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """
+        Validate if the value should be cached
+
+        Args:
+            value: The value to be cached
+
+        Returns:
+            bool: True if the value should be cached
+        """
+        return isinstance(value, dict)  # Cache all dictionaries
+
+    def get_versions(self, type_name: str) -> Dict[str, Any]:
+        """
+        Get version tree for a type
+
+        Args:
+            type_name: Type to get versions for
+
+        Returns:
+            Dict with version information
+        """
+        return self.get(
+            lambda type_name: self._load_version_tree(type_name),
+            type_name=type_name
+        )
+
+    def get_latest_version(self, type_name: str) -> str:
+        """
+        Get the latest version for a given type name.
+
+        Args:
+            type_name: Name of the configuration type
+
+        Returns:
+            Latest version string
+
+        Raises:
+            ValueError: If type not found or no versions available
+        """
+        version_tree = self.get_versions(type_name)
+        if not version_tree or 'latest_version' not in version_tree:
+            raise ValueError(f"No versions found for {type_name}")
+
+        return version_tree['latest_version']
+
+    def get_latest_patch_version(self, type_name: str, major_minor: str) -> str:
+        """
+        Get the latest patch version for a given major.minor version.
+
+        Args:
+            type_name: Name of the configuration type
+            major_minor: Major.minor version (e.g. "1.0")
+
+        Returns:
+            Latest patch version string (e.g. "1.0.3")
+
+        Raises:
+            ValueError: If type not found or no matching versions
+        """
+        version_tree = self.get_versions(type_name)
+        if not version_tree or 'versions' not in version_tree:
+            raise ValueError(f"No versions found for {type_name}")
+
+        # Filter versions that match the major.minor prefix
+        matching_versions = [
+            ver for ver in version_tree['versions'].keys()
+            if ver.startswith(major_minor + '.')
+        ]
+
+        if not matching_versions:
+            raise ValueError(f"No versions found for {type_name} with prefix {major_minor}")
+
+        # Return highest matching version
+        latest_patch = max(matching_versions, key=version.parse)
+        return latest_patch
+
+
+class BaseConfigTypesCacheHandler(CacheHandler[Dict[str, Any]]):
+    """Base handler for configuration types caching"""
+
+    def __init__(self, region, config_type: str):
+        """
+        Args:
+            region: Cache region
+            config_type: Type of configuration (agents, tasks, etc.)
+        """
+        super().__init__(region, f'config_{config_type}_types')
+        self.config_type = config_type
+        self._types_module = None  # Set by subclasses
+        self._config_dir = None  # Set by subclasses
+        self.configure_keys()
+
+    def _to_cache_data(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert the data to a cacheable format"""
+        # For configuration data, we can just return the dictionary as is
+        # since it's already in a serializable format
+        return instance
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+        """Convert cached data back to usable format"""
+        # Similarly, we can return the data directly since it's already
+        # in the format we need
+        return data
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """
+        Validate if the value should be cached
+
+        Args:
+            value: The value to be cached
+
+        Returns:
+            bool: True if the value should be cached
+        """
+        return isinstance(value, dict)  # Cache all dictionaries
+
+    def _load_type_definitions(self) -> Dict[str, Dict[str, str]]:
+        """Load type definitions from the corresponding type_defs module"""
+        if not self._types_module:
+            raise ValueError("_types_module must be set by subclass")
+
+        type_definitions = {
+            type_id: {
+                'name': info['name'],
+                'description': info['description']
+            }
+            for type_id, info in self._types_module.items()
+        }
+
+        return type_definitions
+
+    def get_types(self) -> Dict[str, Dict[str, str]]:
+        """Get dictionary of available types with name and description"""
+        result = self.get(
+            lambda type_name: self._load_type_definitions(),
+            type_name=f'{self.config_type}_types',
+        )
+        return result
+
+
+def create_config_cache_handlers(config_type: str, config_dir: str, types_module: dict) -> tuple:
+    """
+    Factory function to dynamically create the 3 cache handler classes for a given configuration type.
+    The following cache names are created:
+    - <config_type>_config_cache
+    - <config_type>_version_tree_cache
+    - <config_type>_types_cache
+
+
+    Args:
+        config_type: The configuration type (e.g., 'agents', 'tasks').
+        config_dir: The directory where configuration files are stored.
+        types_module: The types module defining the available types for this config.
+
+    Returns:
+        A tuple of dynamically created classes for config, version tree, and types handlers.
+    """
+
+    class ConfigCacheHandler(BaseConfigCacheHandler):
+        handler_name = f"{config_type}_config_cache"
+
+        def __init__(self, region):
+            super().__init__(region, config_type)
+            self._types_module = types_module
+            self._config_dir = config_dir
+
+    class VersionTreeCacheHandler(BaseConfigVersionTreeCacheHandler):
+        handler_name = f"{config_type}_version_tree_cache"
+
+        def __init__(self, region):
+            super().__init__(region, config_type)
+            self._types_module = types_module
+            self._config_dir = config_dir
+
+    class TypesCacheHandler(BaseConfigTypesCacheHandler):
+        handler_name = f"{config_type}_types_cache"
+
+        def __init__(self, region):
+            super().__init__(region, config_type)
+            self._types_module = types_module
+            self._config_dir = config_dir
+
+    return ConfigCacheHandler, VersionTreeCacheHandler, TypesCacheHandler
+
+
+AgentConfigCacheHandler, AgentConfigVersionTreeCacheHandler, AgentConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='agents',
+        config_dir='config/agents',
+        types_module=agent_types.AGENT_TYPES
+    ))
+
+
+TaskConfigCacheHandler, TaskConfigVersionTreeCacheHandler, TaskConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='tasks',
+        config_dir='config/tasks',
+        types_module=task_types.TASK_TYPES
+    ))
+
+
+ToolConfigCacheHandler, ToolConfigVersionTreeCacheHandler, ToolConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='tools',
+        config_dir='config/tools',
+        types_module=tool_types.TOOL_TYPES
+    ))
+
+
+SpecialistConfigCacheHandler, SpecialistConfigVersionTreeCacheHandler, SpecialistConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='specialists',
+        config_dir='config/specialists',
+        types_module=specialist_types.SPECIALIST_TYPES
+    ))
+
+
+RetrieverConfigCacheHandler, RetrieverConfigVersionTreeCacheHandler, RetrieverConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='retrievers',
+        config_dir='config/retrievers',
+        types_module=retriever_types.RETRIEVER_TYPES
+    ))
+
+
+PromptConfigCacheHandler, PromptConfigVersionTreeCacheHandler, PromptConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='prompts',
+        config_dir='config/prompts',
+        types_module=prompt_types.PROMPT_TYPES
+
+    ))
+
+CatalogConfigCacheHandler, CatalogConfigVersionTreeCacheHandler, CatalogConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='catalogs',
+        config_dir='config/catalogs',
+        types_module=catalog_types.CATALOG_TYPES
+
+    ))
+
+
+def register_config_cache_handlers(cache_manager) -> None:
+    cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(TaskConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(TaskConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(TaskConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ToolConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ToolConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ToolConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(SpecialistConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(RetrieverConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(RetrieverConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(RetrieverConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PromptConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PromptConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(PromptConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CatalogConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CatalogConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(CatalogConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
+
+    cache_manager.agents_config_cache.set_version_tree_cache(cache_manager.agents_version_tree_cache)
+    cache_manager.tasks_config_cache.set_version_tree_cache(cache_manager.tasks_version_tree_cache)
+    cache_manager.tools_config_cache.set_version_tree_cache(cache_manager.tools_version_tree_cache)
+    cache_manager.specialists_config_cache.set_version_tree_cache(cache_manager.specialists_version_tree_cache)
+    cache_manager.retrievers_config_cache.set_version_tree_cache(cache_manager.retrievers_version_tree_cache)
+    cache_manager.prompts_config_cache.set_version_tree_cache(cache_manager.prompts_version_tree_cache)

common/utils/cache/crewai_config_processor.py

@@ -0,0 +1,218 @@
+from typing import Dict, Any, Type, TypeVar, List
+from abc import ABC, abstractmethod
+from flask import current_app
+
+from common.extensions import cache_manager, db
+from common.models.interaction import EveAIAgent, EveAITask, EveAITool, Specialist
+from common.utils.cache.crewai_configuration import (
+    ProcessedAgentConfig, ProcessedTaskConfig, ProcessedToolConfig,
+    SpecialistProcessedConfig
+)
+
+T = TypeVar('T')  # For generic model types
+
+
+class BaseCrewAIConfigProcessor:
+    """Base processor for specialist configurations"""
+
+    # Standard mapping between model fields and template placeholders
+    AGENT_FIELD_MAPPING = {
+        'role': 'custom_role',
+        'goal': 'custom_goal',
+        'backstory': 'custom_backstory'
+    }
+
+    TASK_FIELD_MAPPING = {
+        'task_description': 'custom_description',
+        'expected_output': 'custom_expected_output'
+    }
+
+    def __init__(self, tenant_id: int, specialist_id: int):
+        self.tenant_id = tenant_id
+        self.specialist_id = specialist_id
+        self.specialist = self._get_specialist()
+        self.verbose = self._get_verbose_setting()
+
+    def _get_specialist(self) -> Specialist:
+        """Get specialist and verify existence"""
+        specialist = Specialist.query.get(self.specialist_id)
+        if not specialist:
+            raise ValueError(f"Specialist {self.specialist_id} not found")
+        return specialist
+
+    def _get_verbose_setting(self) -> bool:
+        """Get verbose setting from specialist"""
+        return bool(self.specialist.tuning)
+
+    def _get_db_items(self, model_class: Type[T], type_list: List[str]) -> Dict[str, T]:
+        """Get database items of specified type"""
+        items = (model_class.query
+                 .filter_by(specialist_id=self.specialist_id)
+                 .filter(model_class.type.in_(type_list))
+                 .all())
+        return {item.type: item for item in items}
+
+    def _apply_replacements(self, text: str, replacements: Dict[str, str]) -> str:
+        """Apply text replacements to a string"""
+        result = text
+        for key, value in replacements.items():
+            if value is not None:  # Only replace if value exists
+                placeholder = "{" + key + "}"
+                result = result.replace(placeholder, str(value))
+        return result
+
+    def _process_agent_configs(self, specialist_config: Dict[str, Any]) -> Dict[str, ProcessedAgentConfig]:
+        """Process all agent configurations"""
+        agent_configs = {}
+
+        if 'agents' not in specialist_config:
+            return agent_configs
+
+        # Get all DB agents at once
+        agent_types = [agent_def['type'] for agent_def in specialist_config['agents']]
+        db_agents = self._get_db_items(EveAIAgent, agent_types)
+
+        for agent_def in specialist_config['agents']:
+            agent_type = agent_def['type']
+            agent_type_lower = agent_type.lower()
+            db_agent = db_agents.get(agent_type)
+
+            # Get full configuration
+            config = cache_manager.agents_config_cache.get_config(
+                agent_type,
+                agent_def.get('version', '1.0')
+            )
+
+            # Start with YAML values
+            role = config['role']
+            goal = config['goal']
+            backstory = config['backstory']
+
+            # Apply DB values if they exist
+            if db_agent:
+                for model_field, placeholder in self.AGENT_FIELD_MAPPING.items():
+                    value = getattr(db_agent, model_field)
+                    if value:
+                        placeholder_text = "{" + placeholder + "}"
+                        role = role.replace(placeholder_text, value)
+                        goal = goal.replace(placeholder_text, value)
+                        backstory = backstory.replace(placeholder_text, value)
+
+            agent_configs[agent_type_lower] = ProcessedAgentConfig(
+                role=role,
+                goal=goal,
+                backstory=backstory,
+                name=agent_def.get('name') or config.get('name', agent_type_lower),
+                type=agent_type,
+                description=agent_def.get('description') or config.get('description'),
+                verbose=self.verbose
+            )
+
+        return agent_configs
+
+    def _process_task_configs(self, specialist_config: Dict[str, Any]) -> Dict[str, ProcessedTaskConfig]:
+        """Process all task configurations"""
+        task_configs = {}
+
+        if 'tasks' not in specialist_config:
+            return task_configs
+
+        # Get all DB tasks at once
+        task_types = [task_def['type'] for task_def in specialist_config['tasks']]
+        db_tasks = self._get_db_items(EveAITask, task_types)
+
+        for task_def in specialist_config['tasks']:
+            task_type = task_def['type']
+            task_type_lower = task_type.lower()
+            db_task = db_tasks.get(task_type)
+
+            # Get full configuration
+            config = cache_manager.tasks_config_cache.get_config(
+                task_type,
+                task_def.get('version', '1.0')
+            )
+
+            # Start with YAML values
+            task_description = config['task_description']
+            expected_output = config['expected_output']
+
+            # Apply DB values if they exist
+            if db_task:
+                for model_field, placeholder in self.TASK_FIELD_MAPPING.items():
+                    value = getattr(db_task, model_field)
+                    if value:
+                        placeholder_text = "{" + placeholder + "}"
+                        task_description = task_description.replace(placeholder_text, value)
+                        expected_output = expected_output.replace(placeholder_text, value)
+
+            task_configs[task_type_lower] = ProcessedTaskConfig(
+                task_description=task_description,
+                expected_output=expected_output,
+                name=task_def.get('name') or config.get('name', task_type_lower),
+                type=task_type,
+                description=task_def.get('description') or config.get('description'),
+                verbose=self.verbose
+            )
+
+        return task_configs
+
+    def _process_tool_configs(self, specialist_config: Dict[str, Any]) -> Dict[str, ProcessedToolConfig]:
+        """Process all tool configurations"""
+        tool_configs = {}
+
+        if 'tools' not in specialist_config:
+            return tool_configs
+
+        # Get all DB tools at once
+        tool_types = [tool_def['type'] for tool_def in specialist_config['tools']]
+        db_tools = self._get_db_items(EveAITool, tool_types)
+
+        for tool_def in specialist_config['tools']:
+            tool_type = tool_def['type']
+            tool_type_lower = tool_type.lower()
+            db_tool = db_tools.get(tool_type)
+
+            # Get full configuration
+            config = cache_manager.tools_config_cache.get_config(
+                tool_type,
+                tool_def.get('version', '1.0')
+            )
+
+            # Combine configuration
+            tool_config = config.get('configuration', {})
+            if db_tool and db_tool.configuration:
+                tool_config.update(db_tool.configuration)
+
+            tool_configs[tool_type_lower] = ProcessedToolConfig(
+                name=tool_def.get('name') or config.get('name', tool_type_lower),
+                type=tool_type,
+                description=tool_def.get('description') or config.get('description'),
+                configuration=tool_config,
+                verbose=self.verbose
+            )
+
+        return tool_configs
+
+    def process_config(self) -> SpecialistProcessedConfig:
+        """Process complete specialist configuration"""
+        try:
+            # Get full specialist configuration
+            specialist_config = cache_manager.specialists_config_cache.get_config(
+                self.specialist.type,
+                self.specialist.type_version
+            )
+
+            if not specialist_config:
+                raise ValueError(f"No configuration found for {self.specialist.type}")
+
+            # Process all configurations
+            processed_config = SpecialistProcessedConfig(
+                agents=self._process_agent_configs(specialist_config),
+                tasks=self._process_task_configs(specialist_config),
+                tools=self._process_tool_configs(specialist_config)
+            )
+            return processed_config
+
+        except Exception as e:
+            current_app.logger.error(f"Error processing specialist configuration: {e}")
+            raise

common/utils/cache/crewai_configuration.py

@@ -0,0 +1,126 @@
+from dataclasses import dataclass
+from typing import Dict, Any, Optional
+
+
+@dataclass
+class ProcessedAgentConfig:
+    """Processed and ready-to-use agent configuration"""
+    role: str
+    goal: str
+    backstory: str
+    name: str
+    type: str
+    description: Optional[str] = None
+    verbose: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization"""
+        return {
+            'role': self.role,
+            'goal': self.goal,
+            'backstory': self.backstory,
+            'name': self.name,
+            'type': self.type,
+            'description': self.description,
+            'verbose': self.verbose
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedAgentConfig':
+        """Create from dictionary"""
+        return cls(**data)
+
+
+@dataclass
+class ProcessedTaskConfig:
+    """Processed and ready-to-use task configuration"""
+    task_description: str
+    expected_output: str
+    name: str
+    type: str
+    description: Optional[str] = None
+    verbose: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization"""
+        return {
+            'task_description': self.task_description,
+            'expected_output': self.expected_output,
+            'name': self.name,
+            'type': self.type,
+            'description': self.description,
+            'verbose': self.verbose
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedTaskConfig':
+        """Create from dictionary"""
+        return cls(**data)
+
+
+@dataclass
+class ProcessedToolConfig:
+    """Processed and ready-to-use tool configuration"""
+    name: str
+    type: str
+    description: Optional[str] = None
+    configuration: Optional[Dict[str, Any]] = None
+    verbose: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization"""
+        return {
+            'name': self.name,
+            'type': self.type,
+            'description': self.description,
+            'configuration': self.configuration,
+            'verbose': self.verbose
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ProcessedToolConfig':
+        """Create from dictionary"""
+        return cls(**data)
+
+
+@dataclass
+class SpecialistProcessedConfig:
+    """Complete processed configuration for a specialist"""
+    agents: Dict[str, ProcessedAgentConfig]
+    tasks: Dict[str, ProcessedTaskConfig]
+    tools: Dict[str, ProcessedToolConfig]
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert entire configuration to dictionary"""
+        return {
+            'agents': {
+                agent_type: config.to_dict()
+                for agent_type, config in self.agents.items()
+            },
+            'tasks': {
+                task_type: config.to_dict()
+                for task_type, config in self.tasks.items()
+            },
+            'tools': {
+                tool_type: config.to_dict()
+                for tool_type, config in self.tools.items()
+            }
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'SpecialistProcessedConfig':
+        """Create from dictionary"""
+        return cls(
+            agents={
+                agent_type: ProcessedAgentConfig.from_dict(config)
+                for agent_type, config in data['agents'].items()
+            },
+            tasks={
+                task_type: ProcessedTaskConfig.from_dict(config)
+                for task_type, config in data['tasks'].items()
+            },
+            tools={
+                tool_type: ProcessedToolConfig.from_dict(config)
+                for tool_type, config in data['tools'].items()
+            }
+        )

common/utils/cache/crewai_processed_config_cache.py

@@ -0,0 +1,75 @@
+from typing import Dict, Any, Type
+from flask import current_app
+
+from common.utils.cache.base import CacheHandler
+from common.utils.cache.crewai_configuration import SpecialistProcessedConfig
+from common.utils.cache.crewai_config_processor import BaseCrewAIConfigProcessor
+
+
+class CrewAIProcessedConfigCacheHandler(CacheHandler[SpecialistProcessedConfig]):
+    """Handles caching of processed specialist configurations"""
+    handler_name = 'crewai_processed_config_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'crewai_processed_config')
+        self.configure_keys('tenant_id', 'specialist_id')
+
+    def _to_cache_data(self, instance: SpecialistProcessedConfig) -> Dict[str, Any]:
+        """Convert SpecialistProcessedConfig to cache data"""
+        return instance.to_dict()
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> SpecialistProcessedConfig:
+        """Create SpecialistProcessedConfig from cache data"""
+        return SpecialistProcessedConfig.from_dict(data)
+
+    def _should_cache(self, value: Dict[str, Any]) -> bool:
+        """Validate cache data"""
+        required_keys = {'agents', 'tasks', 'tools'}
+        if not all(key in value for key in required_keys):
+            current_app.logger.warning(f'CrewAI Processed Config Cache missing required keys: {required_keys}')
+            return False
+        return bool(value['agents'] or value['tasks'])
+
+    def get_specialist_config(self, tenant_id: int, specialist_id: int) -> SpecialistProcessedConfig:
+        """
+        Get or create processed configuration for a specialist
+
+        Args:
+            tenant_id: Tenant ID
+            specialist_id: Specialist ID
+
+        Returns:
+            Processed specialist configuration
+
+        Raises:
+            ValueError: If specialist not found or processor not configured
+        """
+
+        def creator_func(tenant_id: int, specialist_id: int) -> SpecialistProcessedConfig:
+            # Create processor instance and process config
+            processor = BaseCrewAIConfigProcessor(tenant_id, specialist_id)
+            return processor.process_config()
+
+        return self.get(
+            creator_func,
+            tenant_id=tenant_id,
+            specialist_id=specialist_id
+        )
+
+    def invalidate_tenant_specialist(self, tenant_id: int, specialist_id: int):
+        """Invalidate cache for a specific tenant's specialist"""
+        self.invalidate(
+            tenant_id=tenant_id,
+            specialist_id=specialist_id
+        )
+        current_app.logger.info(
+            f"Invalidated cache for tenant {tenant_id} specialist {specialist_id}"
+        )
+
+
+def register_specialist_cache_handlers(cache_manager) -> None:
+    """Register specialist cache handlers with cache manager"""
+    cache_manager.register_handler(
+        CrewAIProcessedConfigCacheHandler,
+        'eveai_chat_workers'
+    )

common/utils/cache/eveai_cache_manager.py

@@ -3,6 +3,8 @@ from typing import Type
 from flask import Flask
 
 from common.utils.cache.base import CacheHandler
+from common.utils.cache.regions import create_cache_regions
+from common.utils.cache.config_cache import AgentConfigCacheHandler
 
 
 class EveAICacheManager:
@@ -11,29 +13,39 @@ class EveAICacheManager:
     def __init__(self):
         self._regions = {}
         self._handlers = {}
+        self._handler_instances = {}
 
     def init_app(self, app: Flask):
         """Initialize cache regions"""
-        from common.utils.cache.regions import create_cache_regions
         self._regions = create_cache_regions(app)
 
         # Store regions in instance
         for region_name, region in self._regions.items():
             setattr(self, f"{region_name}_region", region)
 
-        # Initialize all registered handlers with their regions
-        for handler_class, region_name in self._handlers.items():
-            region = self._regions[region_name]
-            handler_instance = handler_class(region)
-            handler_name = getattr(handler_class, 'handler_name', None)
-            if handler_name:
-                app.logger.debug(f"{handler_name} is registered")
-                setattr(self, handler_name, handler_instance)
-
-        app.logger.info('Cache regions initialized: ' + ', '.join(self._regions.keys()))
+        app.logger.info(f'Cache regions initialized: {self._regions.keys()}')
 
     def register_handler(self, handler_class: Type[CacheHandler], region: str):
         """Register a cache handler class with its region"""
         if not hasattr(handler_class, 'handler_name'):
             raise ValueError("Cache handler must define handler_name class attribute")
         self._handlers[handler_class] = region
+
+        # Create handler instance
+        region_instance = self._regions[region]
+        handler_instance = handler_class(region_instance)
+        self._handler_instances[handler_class.handler_name] = handler_instance
+
+    def invalidate_region(self, region_name: str):
+        """Invalidate an entire cache region"""
+        if region_name in self._regions:
+            self._regions[region_name].invalidate()
+        else:
+            raise ValueError(f"Unknown cache region: {region_name}")
+
+    def __getattr__(self, name):
+        """Handle dynamic access to registered handlers"""
+        instances = object.__getattribute__(self, '_handler_instances')
+        if name in instances:
+            return instances[name]
+        raise AttributeError(f"'EveAICacheManager' object has no attribute '{name}'")

common/utils/cache/regions.py

@@ -1,4 +1,5 @@
 # common/utils/cache/regions.py
+import time
 
 from dogpile.cache import make_region
 from urllib.parse import urlparse
@@ -36,6 +37,7 @@ def create_cache_regions(app):
     """Initialize all cache regions with app config"""
     redis_config = get_redis_config(app)
     regions = {}
+    startup_time = int(time.time())
 
     # Region for model-related caching (ModelVariables etc)
     model_region = make_region(name='eveai_model').configure(
@@ -61,5 +63,12 @@ def create_cache_regions(app):
     )
     regions['eveai_workers'] = eveai_workers_region
 
+    eveai_config_region = make_region(name='eveai_config').configure(
+        'dogpile.cache.redis',
+        arguments=redis_config,
+        replace_existing_backend=True
+    )
+    regions['eveai_config'] = eveai_config_region
+
     return regions
 

common/utils/celery_utils.py

@@ -58,39 +58,6 @@ def init_celery(celery, app, is_beat=False):
 
     celery.Task = ContextTask
 
-# Original init_celery before updating for beat
-# def init_celery(celery, app):
-#     celery_app.main = app.name
-#     app.logger.debug(f'CELERY_BROKER_URL: {app.config["CELERY_BROKER_URL"]}')
-#     app.logger.debug(f'CELERY_RESULT_BACKEND: {app.config["CELERY_RESULT_BACKEND"]}')
-#     celery_config = {
-#         'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),
-#         'result_backend': app.config.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0'),
-#         'task_serializer': app.config.get('CELERY_TASK_SERIALIZER', 'json'),
-#         'result_serializer': app.config.get('CELERY_RESULT_SERIALIZER', 'json'),
-#         'accept_content': app.config.get('CELERY_ACCEPT_CONTENT', ['json']),
-#         'timezone': app.config.get('CELERY_TIMEZONE', 'UTC'),
-#         'enable_utc': app.config.get('CELERY_ENABLE_UTC', True),
-#         'task_routes': {'eveai_worker.tasks.create_embeddings': {'queue': 'embeddings',
-#                                                                  'routing_key': 'embeddings.create_embeddings'}},
-#     }
-#     celery_app.conf.update(**celery_config)
-#
-#     # Setting up Celery task queues
-#     celery_app.conf.task_queues = (
-#         Queue('default', routing_key='task.#'),
-#         Queue('embeddings', routing_key='embeddings.#', queue_arguments={'x-max-priority': 10}),
-#         Queue('llm_interactions', routing_key='llm_interactions.#', queue_arguments={'x-max-priority': 5}),
-#     )
-#
-#     # Ensuring tasks execute with Flask application context
-#     class ContextTask(celery.Task):
-#         def __call__(self, *args, **kwargs):
-#             with app.app_context():
-#                 return self.run(*args, **kwargs)
-#
-#     celery.Task = ContextTask
-
 
 def make_celery(app_name, config):
     return celery_app

common/utils/config_field_types.py

@@ -652,11 +652,59 @@ def json_to_patterns(json_content: str) -> str:
 def json_to_pattern_list(json_content: str) -> list:
     """Convert JSON patterns list to text area content"""
     try:
-        patterns = json.loads(json_content)
-        if not isinstance(patterns, list):
-            raise ValueError("JSON must contain a list of patterns")
-        # Unescape if needed
-        patterns = [pattern.replace('\\\\', '\\') for pattern in patterns]
-        return patterns
+        if json_content:
+            patterns = json.loads(json_content)
+            if not isinstance(patterns, list):
+                raise ValueError("JSON must contain a list of patterns")
+            # Unescape if needed
+            patterns = [pattern.replace('\\\\', '\\') for pattern in patterns]
+            return patterns
+        else:
+            return []
     except json.JSONDecodeError as e:
         raise ValueError(f"Invalid JSON format: {e}")
+
+
+def normalize_json_field(value: str | dict | None, field_name: str = "JSON field") -> dict:
+    """
+    Normalize a JSON field value to ensure it's a valid dictionary.
+
+    Args:
+        value: The input value which can be:
+            - None (will return empty dict)
+            - String (will be parsed as JSON)
+            - Dict (will be validated and returned)
+        field_name: Name of the field for error messages
+
+    Returns:
+        dict: The normalized JSON data as a Python dictionary
+
+    Raises:
+        ValueError: If the input string is not valid JSON or the input dict contains invalid types
+    """
+    # Handle None case
+    if value is None:
+        return {}
+
+    # Handle dictionary case
+    if isinstance(value, dict):
+        try:
+            # Validate all values are JSON serializable
+            import json
+            json.dumps(value)
+            return value
+        except TypeError as e:
+            raise ValueError(f"{field_name} contains invalid types: {str(e)}")
+
+    # Handle string case
+    if isinstance(value, str):
+        if not value.strip():
+            return {}
+
+        try:
+            import json
+            return json.loads(value)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"{field_name} contains invalid JSON: {str(e)}")
+
+    raise ValueError(f"{field_name} must be a string, dictionary, or None (got {type(value)})")

common/utils/debug_utils.py

@@ -5,58 +5,11 @@ import json
 
 
 def log_request_middleware(app):
-    # @app.before_request
-    # def log_request_info():
-    #     start_time = time.time()
-    #     app.logger.debug(f"Request URL: {request.url}")
-    #     app.logger.debug(f"Request Method: {request.method}")
-    #     app.logger.debug(f"Request Headers: {request.headers}")
-    #     app.logger.debug(f"Time taken for logging request info: {time.time() - start_time} seconds")
-    #     try:
-    #         app.logger.debug(f"Request Body: {request.get_data()}")
-    #     except Exception as e:
-    #         app.logger.error(f"Error reading request body: {e}")
-    #     app.logger.debug(f"Time taken for logging request body: {time.time() - start_time} seconds")
-
-    # @app.before_request
-    # def check_csrf_token():
-    #     start_time = time.time()
-    #     if request.method == "POST":
-    #         csrf_token = request.form.get("csrf_token")
-    #         app.logger.debug(f"CSRF Token: {csrf_token}")
-    #     app.logger.debug(f"Time taken for logging CSRF token: {time.time() - start_time} seconds")
-
-    # @app.before_request
-    # def log_user_info():
-    #     if current_user and current_user.is_authenticated:
-    #         app.logger.debug(f"Before: User ID: {current_user.id}")
-    #         app.logger.debug(f"Before: User Email: {current_user.email}")
-    #         app.logger.debug(f"Before: User Roles: {current_user.roles}")
-    #     else:
-    #         app.logger.debug("After: No user logged in")
 
     @app.before_request
     def log_session_state_before():
         pass
 
-    # @app.after_request
-    # def log_response_info(response):
-    #     start_time = time.time()
-    #     app.logger.debug(f"Response Status: {response.status}")
-    #     app.logger.debug(f"Response Headers: {response.headers}")
-    #
-    #     app.logger.debug(f"Time taken for logging response info: {time.time() - start_time} seconds")
-    #     return response
-
-    # @app.after_request
-    # def log_user_after_request(response):
-    #     if current_user and current_user.is_authenticated:
-    #         app.logger.debug(f"After: User ID: {current_user.id}")
-    #         app.logger.debug(f"after: User Email: {current_user.email}")
-    #         app.logger.debug(f"After: User Roles: {current_user.roles}")
-    #     else:
-    #         app.logger.debug("After: No user logged in")
-
     @app.after_request
     def log_session_state_after(response):
         return response
@@ -149,8 +102,3 @@ def register_request_debugger(app):
         # Format the debug info as a pretty-printed JSON string with indentation
         formatted_debug_info = json.dumps(debug_info, indent=2, sort_keys=True)
 
-        # Log everything in a single statement
-        app.logger.debug(
-            "Request Debug Information\n",
-            extra={"request_debug\n": formatted_debug_info}
-        )

common/utils/document_utils.py

@@ -7,13 +7,15 @@ from common.models.document import Document, DocumentVersion, Catalog
 from common.extensions import db, minio_client
 from common.utils.celery_utils import current_celery
 from flask import current_app
-from flask_security import current_user
 import requests
-from urllib.parse import urlparse, unquote, urlunparse
+from urllib.parse import urlparse, unquote, urlunparse, parse_qs
 import os
+
+from .config_field_types import normalize_json_field
 from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
                                EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
 from ..models.user import Tenant
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
 
 
 def create_document_stack(api_input, file, filename, extension, tenant_id):
@@ -88,10 +90,10 @@ def create_version_for_document(document, tenant_id, url, sub_file_type,  langua
         new_doc_vers.user_context = user_context
 
     if user_metadata != '' and user_metadata is not None:
-        new_doc_vers.user_metadata = user_metadata
+        new_doc_vers.user_metadata = normalize_json_field(user_metadata, "user_metadata")
 
     if catalog_properties != '' and catalog_properties is not None:
-        new_doc_vers.catalog_properties = catalog_properties
+        new_doc_vers.catalog_properties = normalize_json_field(catalog_properties, "catalog_properties")
 
     if sub_file_type != '':
         new_doc_vers.sub_file_type = sub_file_type
@@ -134,35 +136,6 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id):
         raise
 
 
-def set_logging_information(obj, timestamp):
-    obj.created_at = timestamp
-    obj.updated_at = timestamp
-
-    user_id = get_current_user_id()
-    if user_id:
-        obj.created_by = user_id
-        obj.updated_by = user_id
-
-
-def update_logging_information(obj, timestamp):
-    obj.updated_at = timestamp
-
-    user_id = get_current_user_id()
-    if user_id:
-        obj.updated_by = user_id
-
-
-def get_current_user_id():
-    try:
-        if current_user and current_user.is_authenticated:
-            return current_user.id
-        else:
-            return None
-    except Exception:
-        # This will catch any errors if current_user is not available (e.g., in API context)
-        return None
-
-
 def get_extension_from_content_type(content_type):
     content_type_map = {
         'text/html': 'html',
@@ -209,6 +182,24 @@ def process_url(url, tenant_id):
     return file_content, filename, extension
 
 
+def clean_url(url):
+    tracking_params = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
+                       "hsa_acc", "hsa_cam", "hsa_grp", "hsa_ad", "hsa_src", "hsa_tgt", "hsa_kw",
+                       "hsa_mt", "hsa_net", "hsa_ver", "gad_source", "gbraid"}
+
+    parsed_url = urlparse(url)
+    query_params = parse_qs(parsed_url.query)
+
+    # Remove tracking params
+    clean_params = {k: v for k, v in query_params.items() if k not in tracking_params}
+
+    # Reconstruct the URL
+    clean_query = "&".join(f"{k}={v[0]}" for k, v in clean_params.items()) if clean_params else ""
+    cleaned_url = urlunparse(parsed_url._replace(query=clean_query))
+
+    return cleaned_url
+
+
 def start_embedding_task(tenant_id, doc_vers_id):
     task = current_celery.send_task('create_embeddings',
                                     args=[tenant_id, doc_vers_id,],
@@ -262,7 +253,8 @@ def edit_document_version(tenant_id, version_id, user_context, catalog_propertie
     if not doc_vers:
         raise EveAIInvalidDocumentVersion(tenant_id, version_id)
     doc_vers.user_context = user_context
-    doc_vers.catalog_properties = catalog_properties
+    doc_vers.catalog_properties = normalize_json_field(catalog_properties, "catalog_properties")
+
     update_logging_information(doc_vers, dt.now(tz.utc))
 
     try:
@@ -319,6 +311,56 @@ def refresh_document_with_info(doc_id, tenant_id, api_input):
     return new_doc_vers, task.id
 
 
+def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: bytes, api_input: dict) -> tuple:
+    """
+    Refresh document with new content
+
+    Args:
+        doc_id: Document ID
+        tenant_id: Tenant ID
+        file_content: New file content
+        api_input: Additional document information
+
+    Returns:
+        Tuple of (new_version, task_id)
+    """
+    doc = Document.query.get(doc_id)
+    if not doc:
+        raise EveAIInvalidDocument(tenant_id, doc_id)
+
+    old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
+
+    # Create new version with same file type as original
+    extension = old_doc_vers.file_type
+
+    new_doc_vers = create_version_for_document(
+        doc, tenant_id,
+        '',  # No URL for content-based updates
+        old_doc_vers.sub_file_type,
+        api_input.get('language', old_doc_vers.language),
+        api_input.get('user_context', old_doc_vers.user_context),
+        api_input.get('user_metadata', old_doc_vers.user_metadata),
+        api_input.get('catalog_properties', old_doc_vers.catalog_properties),
+    )
+
+    try:
+        db.session.add(new_doc_vers)
+        db.session.commit()
+    except SQLAlchemyError as e:
+        db.session.rollback()
+        return None, str(e)
+
+    # Upload new content
+    upload_file_for_version(new_doc_vers, file_content, extension, tenant_id)
+
+    # Start embedding task
+    task = current_celery.send_task('create_embeddings', args=[tenant_id, new_doc_vers.id], queue='embeddings')
+    current_app.logger.info(f'Embedding creation started for document {doc_id} on version {new_doc_vers.id} '
+                            f'with task id: {task.id}.')
+
+    return new_doc_vers, task.id
+
+
 # Update the existing refresh_document function to use the new refresh_document_with_info
 def refresh_document(doc_id, tenant_id):
     current_app.logger.info(f'Refreshing document {doc_id}')
@@ -343,7 +385,6 @@ def mark_tenant_storage_dirty(tenant_id):
 
 
 def cope_with_local_url(url):
-    current_app.logger.debug(f'Incomming URL: {url}')
     parsed_url = urlparse(url)
     # Check if this is an internal WordPress URL (TESTING) and rewrite it
     if parsed_url.netloc in [current_app.config['EXTERNAL_WORDPRESS_BASE_URL']]:
@@ -352,7 +393,6 @@ def cope_with_local_url(url):
             netloc=f"{current_app.config['WORDPRESS_HOST']}:{current_app.config['WORDPRESS_PORT']}"
         )
         url = urlunparse(parsed_url)
-        current_app.logger.debug(f'Translated Wordpress URL to: {url}')
 
     return url
 
@@ -411,55 +451,3 @@ def lookup_document(tenant_id: int, lookup_criteria: dict, metadata_type: str) -
             "Error during document lookup",
             status_code=500
         )
-
-
-# Add to common/utils/document_utils.py
-
-def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: bytes, api_input: dict) -> tuple:
-    """
-    Refresh document with new content
-
-    Args:
-        doc_id: Document ID
-        tenant_id: Tenant ID
-        file_content: New file content
-        api_input: Additional document information
-
-    Returns:
-        Tuple of (new_version, task_id)
-    """
-    doc = Document.query.get(doc_id)
-    if not doc:
-        raise EveAIInvalidDocument(tenant_id, doc_id)
-
-    old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
-
-    # Create new version with same file type as original
-    extension = old_doc_vers.file_type
-
-    new_doc_vers = create_version_for_document(
-        doc, tenant_id,
-        '',  # No URL for content-based updates
-        old_doc_vers.sub_file_type,
-        api_input.get('language', old_doc_vers.language),
-        api_input.get('user_context', old_doc_vers.user_context),
-        api_input.get('user_metadata', old_doc_vers.user_metadata),
-        api_input.get('catalog_properties', old_doc_vers.catalog_properties),
-    )
-
-    try:
-        db.session.add(new_doc_vers)
-        db.session.commit()
-    except SQLAlchemyError as e:
-        db.session.rollback()
-        return None, str(e)
-
-    # Upload new content
-    upload_file_for_version(new_doc_vers, file_content, extension, tenant_id)
-
-    # Start embedding task
-    task = current_celery.send_task('create_embeddings', args=[tenant_id, new_doc_vers.id], queue='embeddings')
-    current_app.logger.info(f'Embedding creation started for document {doc_id} on version {new_doc_vers.id} '
-                            f'with task id: {task.id}.')
-
-    return new_doc_vers, task.id

common/utils/eveai_exceptions.py

@@ -124,4 +124,15 @@ class EveAISocketInputException(EveAIException):
     """Raised when a socket call receives an invalid payload"""
 
     def __init__(self, message, status_code=400, payload=None):
-        super.__init__(message, status_code, payload)
+        super.__init__(message, status_code, payload)
+
+
+class EveAIInvalidEmbeddingModel(EveAIException):
+    """Raised when no or an invalid embedding model is provided in the catalog"""
+
+    def __init__(self, tenant_id, catalog_id, status_code=400, payload=None):
+        self.tenant_id = tenant_id
+        self.catalog_id = catalog_id
+        # Construct the message dynamically
+        message = f"Tenant with ID '{tenant_id}' has no or an invalid embedding model in Catalog {catalog_id}."
+        super().__init__(message, status_code, payload)

common/utils/execution_progress.py

@@ -0,0 +1,112 @@
+# common/utils/execution_progress.py
+from datetime import datetime as dt, timezone as tz
+from typing import Generator
+from redis import Redis, RedisError
+import json
+from flask import current_app
+
+
+class ExecutionProgressTracker:
+    """Tracks progress of specialist executions using Redis"""
+
+    def __init__(self):
+        try:
+            redis_url = current_app.config['SPECIALIST_EXEC_PUBSUB']
+
+            self.redis = Redis.from_url(redis_url, socket_timeout=5)
+            # Test the connection
+            self.redis.ping()
+
+            self.expiry = 3600  # 1 hour expiry
+        except RedisError as e:
+            current_app.logger.error(f"Failed to connect to Redis: {str(e)}")
+            raise
+        except Exception as e:
+            current_app.logger.error(f"Unexpected error during Redis initialization: {str(e)}")
+            raise
+
+    def _get_key(self, execution_id: str) -> str:
+        return f"specialist_execution:{execution_id}"
+
+    def send_update(self, ctask_id: str, processing_type: str, data: dict):
+        """Send an update about execution progress"""
+        try:
+            key = self._get_key(ctask_id)
+
+            # First verify Redis is still connected
+            try:
+                self.redis.ping()
+            except RedisError:
+                current_app.logger.error("Lost Redis connection. Attempting to reconnect...")
+                self.__init__()  # Reinitialize connection
+
+            update = {
+                'processing_type': processing_type,
+                'data': data,
+                'timestamp': dt.now(tz=tz.utc)
+            }
+
+            # Log initial state
+            try:
+                orig_len = self.redis.llen(key)
+
+                # Try to serialize the update and check the result
+                try:
+                    serialized_update = json.dumps(update, default=str)  # Add default handler for datetime
+                except TypeError as e:
+                    current_app.logger.error(f"Failed to serialize update: {str(e)}")
+                    raise
+
+                # Store update in list with pipeline for atomicity
+                with self.redis.pipeline() as pipe:
+                    pipe.rpush(key, serialized_update)
+                    pipe.publish(key, serialized_update)
+                    pipe.expire(key, self.expiry)
+                    results = pipe.execute()
+
+                new_len = self.redis.llen(key)
+
+                if new_len <= orig_len:
+                    current_app.logger.error(
+                        f"List length did not increase as expected. Original: {orig_len}, New: {new_len}")
+
+            except RedisError as e:
+                current_app.logger.error(f"Redis operation failed: {str(e)}")
+                raise
+
+        except Exception as e:
+            current_app.logger.error(f"Unexpected error in send_update: {str(e)}, type: {type(e)}")
+            raise
+
+    def get_updates(self, ctask_id: str) -> Generator[str, None, None]:
+        key = self._get_key(ctask_id)
+        pubsub = self.redis.pubsub()
+        pubsub.subscribe(key)
+
+        try:
+            # First yield any existing updates
+            length = self.redis.llen(key)
+            if length > 0:
+                updates = self.redis.lrange(key, 0, -1)
+                for update in updates:
+                    update_data = json.loads(update.decode('utf-8'))
+                    # Use processing_type for the event
+                    yield f"event: {update_data['processing_type']}\n"
+                    yield f"data: {json.dumps(update_data)}\n\n"
+
+            # Then listen for new updates
+            while True:
+                message = pubsub.get_message(timeout=30)  # message['type'] is Redis pub/sub type
+                if message is None:
+                    yield ": keepalive\n\n"
+                    continue
+
+                if message['type'] == 'message':  # This is Redis pub/sub type
+                    update_data = json.loads(message['data'].decode('utf-8'))
+                    yield f"data: {message['data'].decode('utf-8')}\n\n"
+
+                    # Check processing_type for completion
+                    if update_data['processing_type'] in ['Task Complete', 'Task Error']:
+                        break
+        finally:
+            pubsub.unsubscribe()

common/utils/minio_utils.py

@@ -33,6 +33,9 @@ class MinioClient:
     def generate_object_name(self, document_id, language, version_id, filename):
         return f"{document_id}/{language}/{version_id}/{filename}"
 
+    def generate_asset_name(self, asset_version_id, file_name, content_type):
+        return f"assets/{asset_version_id}/{file_name}.{content_type}"
+
     def upload_document_file(self, tenant_id, document_id, language, version_id, filename, file_data):
         bucket_name = self.generate_bucket_name(tenant_id)
         object_name = self.generate_object_name(document_id, language, version_id, filename)
@@ -54,6 +57,26 @@ class MinioClient:
         except S3Error as err:
             raise Exception(f"Error occurred while uploading file: {err}")
 
+    def upload_asset_file(self, bucket_name, asset_version_id, file_name, file_type, file_data):
+        object_name = self.generate_asset_name(asset_version_id, file_name, file_type)
+
+        try:
+            if isinstance(file_data, FileStorage):
+                file_data = file_data.read()
+            elif isinstance(file_data, io.BytesIO):
+                file_data = file_data.getvalue()
+            elif isinstance(file_data, str):
+                file_data = file_data.encode('utf-8')
+            elif not isinstance(file_data, bytes):
+                raise TypeError('Unsupported file type. Expected FileStorage, BytesIO, str, or bytes.')
+
+            self.client.put_object(
+                bucket_name, object_name, io.BytesIO(file_data), len(file_data)
+            )
+            return object_name, len(file_data)
+        except S3Error as err:
+            raise Exception(f"Error occurred while uploading asset: {err}")
+
     def download_document_file(self, tenant_id, bucket_name, object_name):
         try:
             response = self.client.get_object(bucket_name, object_name)

common/utils/model_logging_utils.py

@@ -0,0 +1,30 @@
+from flask_security import current_user
+
+
+def set_logging_information(obj, timestamp):
+    obj.created_at = timestamp
+    obj.updated_at = timestamp
+
+    user_id = get_current_user_id()
+    if user_id:
+        obj.created_by = user_id
+        obj.updated_by = user_id
+
+
+def update_logging_information(obj, timestamp):
+    obj.updated_at = timestamp
+
+    user_id = get_current_user_id()
+    if user_id:
+        obj.updated_by = user_id
+
+
+def get_current_user_id():
+    try:
+        if current_user and current_user.is_authenticated:
+            return current_user.id
+        else:
+            return None
+    except Exception:
+        # This will catch any errors if current_user is not available (e.g., in API context)
+        return None

common/utils/model_utils.py

@@ -1,23 +1,27 @@
 import os
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, Tuple
 
 import langcodes
+from langchain_core.language_models import BaseChatModel
 
 from common.langchain.llm_metrics_handler import LLMMetricsHandler
-from common.langchain.templates.template_manager import TemplateManager
-from langchain_openai import OpenAIEmbeddings, ChatOpenAI, OpenAI
+from langchain_openai import ChatOpenAI
 from langchain_anthropic import ChatAnthropic
+from langchain_mistralai import ChatMistralAI
 from flask import current_app
-from datetime import datetime as dt, timezone as tz
 
-from common.langchain.tracked_openai_embeddings import TrackedOpenAIEmbeddings
+from common.eveai_model.tracked_mistral_embeddings import TrackedMistralAIEmbeddings
 from common.langchain.tracked_transcription import TrackedOpenAITranscription
 from common.models.user import Tenant
-from common.utils.cache.base import CacheHandler
 from config.model_config import MODEL_CONFIG
-from common.extensions import template_manager, cache_manager
-from common.models.document import EmbeddingLargeOpenAI, EmbeddingSmallOpenAI
-from common.utils.eveai_exceptions import EveAITenantNotFound
+from common.extensions import template_manager
+from common.models.document import EmbeddingMistral
+from common.utils.eveai_exceptions import EveAITenantNotFound, EveAIInvalidEmbeddingModel
+from crewai import LLM
+
+embedding_llm_model_cache: Dict[Tuple[str, float], BaseChatModel] = {}
+crewai_llm_model_cache: Dict[Tuple[str, float], LLM] = {}
+llm_metrics_handler = LLMMetricsHandler()
 
 
 def create_language_template(template: str, language: str) -> str:
@@ -55,6 +59,82 @@ def replace_variable_in_template(template: str, variable: str, value: str) -> st
     return template.replace(variable, value or "")
 
 
+def get_embedding_model_and_class(tenant_id, catalog_id, full_embedding_name="mistral.mistral-embed"):
+    """
+        Retrieve the embedding model and embedding model class to store Embeddings
+
+        Args:
+            tenant_id: ID of the tenant
+            catalog_id: ID of the catalog
+            full_embedding_name: The full name of the embedding model: <provider>.<model>
+
+        Returns:
+            embedding_model, embedding_model_class
+        """
+    embedding_provider, embedding_model_name = full_embedding_name.split('.')
+
+    # Calculate the embedding model to be used
+    if embedding_provider == "mistral":
+        api_key = current_app.config['MISTRAL_API_KEY']
+        embedding_model = TrackedMistralAIEmbeddings(
+            model=embedding_model_name
+        )
+    else:
+        raise EveAIInvalidEmbeddingModel(tenant_id, catalog_id)
+
+    # Calculate the Embedding Model Class to be used to store embeddings
+    if embedding_model_name == "mistral-embed":
+        embedding_model_class = EmbeddingMistral
+    else:
+        raise EveAIInvalidEmbeddingModel(tenant_id, catalog_id)
+
+    return embedding_model, embedding_model_class
+
+
+def get_embedding_llm(full_model_name='mistral.mistral-small-latest', temperature=0.3):
+    llm = embedding_llm_model_cache.get((full_model_name, temperature))
+    if not llm:
+        llm_provider, llm_model_name = full_model_name.split('.')
+        if llm_provider == "openai":
+            llm = ChatOpenAI(
+                api_key=current_app.config['OPENAI_API_KEY'],
+                model=llm_model_name,
+                temperature=temperature,
+                callbacks=[llm_metrics_handler]
+            )
+        elif llm_provider == "mistral":
+            llm = ChatMistralAI(
+                api_key=current_app.config['MISTRAL_API_KEY'],
+                model=llm_model_name,
+                temperature=temperature,
+                callbacks=[llm_metrics_handler]
+            )
+        embedding_llm_model_cache[(full_model_name, temperature)] = llm
+
+    return llm
+
+
+def get_crewai_llm(full_model_name='mistral.mistral-large-latest', temperature=0.3):
+    llm = crewai_llm_model_cache.get((full_model_name, temperature))
+    if not llm:
+        llm_provider, llm_model_name = full_model_name.split('.')
+        crew_full_model_name = f"{llm_provider}/{llm_model_name}"
+        api_key = None
+        if llm_provider == "openai":
+            api_key = current_app.config['OPENAI_API_KEY']
+        elif llm_provider == "mistral":
+            api_key = current_app.config['MISTRAL_API_KEY']
+
+        llm = LLM(
+            model=crew_full_model_name,
+            temperature=temperature,
+            api_key=api_key
+        )
+        crewai_llm_model_cache[(full_model_name, temperature)] = llm
+
+    return llm
+
+
 class ModelVariables:
     """Manages model-related variables and configurations"""
 
@@ -63,15 +143,13 @@ class ModelVariables:
         Initialize ModelVariables with tenant and optional template manager
 
         Args:
-            tenant: Tenant instance
-            template_manager: Optional TemplateManager instance
+            tenant_id: Tenant instance
+            variables: Optional variables
         """
         current_app.logger.info(f'Model variables initialized with tenant {tenant_id} and variables \n{variables}')
         self.tenant_id = tenant_id
         self._variables = variables if variables is not None else self._initialize_variables()
         current_app.logger.info(f'Model _variables initialized to {self._variables}')
-        self._embedding_model = None
-        self._embedding_model_class = None
         self._llm_instances = {}
         self.llm_metrics_handler = LLMMetricsHandler()
         self._transcription_model = None
@@ -85,7 +163,6 @@ class ModelVariables:
             raise EveAITenantNotFound(self.tenant_id)
 
         # Set model providers
-        variables['embedding_provider'], variables['embedding_model'] = tenant.embedding_model.split('.')
         variables['llm_provider'], variables['llm_model'] = tenant.llm_model.split('.')
         variables['llm_full_model'] = tenant.llm_model
 
@@ -102,28 +179,6 @@ class ModelVariables:
 
         return variables
 
-    @property
-    def embedding_model(self):
-        """Get the embedding model instance"""
-        if self._embedding_model is None:
-            api_key = os.getenv('OPENAI_API_KEY')
-            self._embedding_model = TrackedOpenAIEmbeddings(
-                api_key=api_key,
-                model=self._variables['embedding_model']
-            )
-        return self._embedding_model
-
-    @property
-    def embedding_model_class(self):
-        """Get the embedding model class"""
-        if self._embedding_model_class is None:
-            if self._variables['embedding_model'] == 'text-embedding-3-large':
-                self._embedding_model_class = EmbeddingLargeOpenAI
-            else:   # text-embedding-3-small
-                self._embedding_model_class = EmbeddingSmallOpenAI
-
-        return self._embedding_model_class
-
     @property
     def annotation_chunk_length(self):
         return self._variables['annotation_chunk_length']
@@ -227,62 +282,6 @@ class ModelVariables:
             raise
 
 
-class ModelVariablesCacheHandler(CacheHandler[ModelVariables]):
-    handler_name = 'model_vars_cache'  # Used to access handler instance from cache_manager
-
-    def __init__(self, region):
-        super().__init__(region, 'model_variables')
-        self.configure_keys('tenant_id')
-        self.subscribe_to_model('Tenant', ['tenant_id'])
-
-    def to_cache_data(self, instance: ModelVariables) -> Dict[str, Any]:
-        return {
-            'tenant_id': instance.tenant_id,
-            'variables': instance._variables,
-            'last_updated': dt.now(tz=tz.utc).isoformat()
-        }
-
-    def from_cache_data(self, data: Dict[str, Any], tenant_id: int, **kwargs) -> ModelVariables:
-        instance = ModelVariables(tenant_id, data.get('variables'))
-        return instance
-
-    def should_cache(self, value: Dict[str, Any]) -> bool:
-        required_fields = {'tenant_id', 'variables'}
-        return all(field in value for field in required_fields)
-
-
-# Register the handler with the cache manager
-cache_manager.register_handler(ModelVariablesCacheHandler, 'eveai_model')
-
-
 # Helper function to get cached model variables
 def get_model_variables(tenant_id: int) -> ModelVariables:
-    return cache_manager.model_vars_cache.get(
-        lambda tenant_id: ModelVariables(tenant_id),    # function to create ModelVariables if required
-        tenant_id=tenant_id
-    )
-
-# Written in a long format, without lambda
-# def get_model_variables(tenant_id: int) -> ModelVariables:
-#     """
-#     Get ModelVariables instance, either from cache or newly created
-#
-#     Args:
-#         tenant_id: The tenant's ID
-#
-#     Returns:
-#         ModelVariables: Instance with either cached or fresh data
-#
-#     Raises:
-#         TenantNotFoundError: If tenant doesn't exist
-#         CacheStateError: If cached data is invalid
-#     """
-#
-#     def create_new_instance(tenant_id: int) -> ModelVariables:
-#         """Creator function that's called when cache miss occurs"""
-#         return ModelVariables(tenant_id)  # This will initialize fresh variables
-#
-#     return cache_manager.model_vars_cache.get(
-#         create_new_instance,  # Function to create new instance if needed
-#         tenant_id=tenant_id  # Parameters passed to both get() and create_new_instance
-#     )
+    return ModelVariables(tenant_id=tenant_id)

common/utils/performance_monitoring.py

@@ -0,0 +1,59 @@
+import time
+import threading
+from contextlib import contextmanager
+from functools import wraps
+from prometheus_client import Counter, Histogram, Summary, start_http_server, Gauge
+from flask import current_app, g, request, Flask
+
+
+class EveAIMetrics:
+    """
+    Central class for Prometheus metrics infrastructure.
+    This class initializes the Prometheus HTTP server and provides
+    shared functionality for metrics across components.
+
+    Component-specific metrics should be defined in their respective modules.
+    """
+
+    def __init__(self, app: Flask = None):
+        self.app = app
+        self._metrics_server_started = False
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app: Flask):
+        """Initialize metrics with Flask app and start Prometheus server"""
+        self.app = app
+        self._start_metrics_server()
+
+    def _start_metrics_server(self):
+        """Start the Prometheus metrics HTTP server if not already running"""
+        if not self._metrics_server_started:
+            try:
+                metrics_port = self.app.config.get('PROMETHEUS_PORT', 8000)
+                start_http_server(metrics_port)
+                self.app.logger.info(f"Prometheus metrics server started on port {metrics_port}")
+                self._metrics_server_started = True
+            except Exception as e:
+                self.app.logger.error(f"Failed to start metrics server: {e}")
+
+    @staticmethod
+    def get_standard_buckets():
+        """
+        Return the standard duration buckets for histogram metrics.
+        Components should use these for consistency across the system.
+        """
+        return [0.1, 0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 240, 360, float('inf')]
+
+    @staticmethod
+    def sanitize_label_values(labels_dict):
+        """
+        Convert all label values to strings as required by Prometheus.
+
+        Args:
+            labels_dict: Dictionary of label name to label value
+
+        Returns:
+            Dictionary with all values converted to strings
+        """
+        return {k: str(v) if v is not None else "" for k, v in labels_dict.items()}

common/utils/pydantic_utils.py

@@ -0,0 +1,78 @@
+from pydantic import BaseModel, Field
+from typing import Any, Dict, List, Optional, Type, Union
+
+
+def flatten_pydantic_model(model: BaseModel, merge_strategy: Dict[str, str] = {}) -> Dict[str, Any]:
+    """
+    Flattens a nested Pydantic model by bringing all attributes to the highest level.
+
+    :param model: Pydantic model instance to be flattened.
+    :param merge_strategy: Dictionary defining how to handle duplicate attributes.
+    :return: Flattened dictionary representation of the model.
+    """
+    flat_dict = {}
+
+    def recursive_flatten(obj: BaseModel, parent_key=""):
+        for field_name, value in obj.model_dump(exclude_unset=True, by_alias=True).items():
+            new_key = field_name  # Maintain original field names
+
+            if isinstance(value, BaseModel):
+                # Recursively flatten nested models
+                recursive_flatten(value, new_key)
+            elif isinstance(value, list) and all(isinstance(i, BaseModel) for i in value):
+                # If it's a list of Pydantic models, flatten each element
+                for item in value:
+                    recursive_flatten(item, new_key)
+            else:
+                if new_key in flat_dict and new_key in merge_strategy:
+                    # Apply merge strategy
+                    if merge_strategy[new_key] == "add":
+                        if isinstance(flat_dict[new_key], list) and isinstance(value, list):
+                            flat_dict[new_key] += value  # Concatenate lists
+                        elif isinstance(flat_dict[new_key], (int, float)) and isinstance(value, (int, float)):
+                            flat_dict[new_key] += value  # Sum numbers
+                        elif isinstance(flat_dict[new_key], str) and isinstance(value, str):
+                            flat_dict[new_key] += "\n" + value  # Concatenate strings
+                    elif merge_strategy[new_key] == "first":
+                        pass  # Keep the first occurrence
+                    elif merge_strategy[new_key] == "last":
+                        flat_dict[new_key] = value
+                else:
+                    flat_dict[new_key] = value
+
+    recursive_flatten(model)
+    return flat_dict
+
+
+def merge_dicts(base_dict: Dict[str, Any], new_data: Union[Dict[str, Any], BaseModel], merge_strategy: Dict[str, str]) \
+        -> Dict[str, Any]:
+    """
+    Merges a Pydantic model (or dictionary) into an existing dictionary based on a merge strategy.
+
+    :param base_dict: The base dictionary to merge into.
+    :param new_data: The new Pydantic model or dictionary to merge.
+    :param merge_strategy: Dict defining how to merge duplicate attributes.
+    :return: Updated dictionary after merging.
+    """
+    if isinstance(new_data, BaseModel):
+        new_data = flatten_pydantic_model(new_data)  # Convert Pydantic model to dict
+
+    for key, value in new_data.items():
+        if key in base_dict and key in merge_strategy:
+            strategy = merge_strategy[key]
+
+            if strategy == "add":
+                if isinstance(base_dict[key], list) and isinstance(value, list):
+                    base_dict[key] += value  # Concatenate lists
+                elif isinstance(base_dict[key], (int, float)) and isinstance(value, (int, float)):
+                    base_dict[key] += value  # Sum numbers
+                elif isinstance(base_dict[key], str) and isinstance(value, str):
+                    base_dict[key] += " " + value  # Concatenate strings
+            elif strategy == "first":
+                pass  # Keep the first occurrence (do nothing)
+            elif strategy == "last":
+                base_dict[key] = value  # Always overwrite with latest value
+        else:
+            base_dict[key] = value  # Add new field
+
+    return base_dict

common/utils/security.py

@@ -13,14 +13,12 @@ def set_tenant_session_data(sender, user, **kwargs):
     tenant = Tenant.query.filter_by(id=user.tenant_id).first()
     session['tenant'] = tenant.to_dict()
     session['default_language'] = tenant.default_language
-    session['default_embedding_model'] = tenant.embedding_model
     session['default_llm_model'] = tenant.llm_model
 
 
 def clear_tenant_session_data(sender, user, **kwargs):
     session.pop('tenant', None)
     session.pop('default_language', None)
-    session.pop('default_embedding_model', None)
     session.pop('default_llm_model', None)
 
 

common/utils/specialist_utils.py

@@ -0,0 +1,196 @@
+from datetime import datetime as dt, timezone as tz
+from typing import Optional, Dict, Any
+from flask import current_app
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db, cache_manager
+from common.models.interaction import (
+    Specialist, EveAIAgent, EveAITask, EveAITool
+)
+from common.utils.model_logging_utils import set_logging_information, update_logging_information
+
+
+def initialize_specialist(specialist_id: int, specialist_type: str, specialist_version: str):
+    """
+    Initialize an agentic specialist by creating all its components based on configuration.
+
+    Args:
+        specialist_id: ID of the specialist to initialize
+        specialist_type: Type of the specialist
+        specialist_version: Version of the specialist type to use
+
+    Raises:
+        ValueError: If specialist not found or invalid configuration
+        SQLAlchemyError: If database operations fail
+    """
+    config = cache_manager.specialists_config_cache.get_config(specialist_type, specialist_version)
+    if not config:
+        raise ValueError(f"No configuration found for {specialist_type} version {specialist_version}")
+    if config['framework'] == 'langchain':
+        pass    # Langchain does not require additional items to be initialized. All configuration is in the specialist.
+
+    specialist = Specialist.query.get(specialist_id)
+    if not specialist:
+        raise ValueError(f"Specialist with ID {specialist_id} not found")
+
+    if config['framework'] == 'crewai':
+        initialize_crewai_specialist(specialist, config)
+
+
+def initialize_crewai_specialist(specialist: Specialist, config: Dict[str, Any]):
+    timestamp = dt.now(tz=tz.utc)
+
+    try:
+        # Initialize agents
+        if 'agents' in config:
+            for agent_config in config['agents']:
+                _create_agent(
+                    specialist_id=specialist.id,
+                    agent_type=agent_config['type'],
+                    agent_version=agent_config['version'],
+                    name=agent_config.get('name'),
+                    description=agent_config.get('description'),
+                    timestamp=timestamp
+                )
+
+        # Initialize tasks
+        if 'tasks' in config:
+            for task_config in config['tasks']:
+                _create_task(
+                    specialist_id=specialist.id,
+                    task_type=task_config['type'],
+                    task_version=task_config['version'],
+                    name=task_config.get('name'),
+                    description=task_config.get('description'),
+                    timestamp=timestamp
+                )
+
+        # Initialize tools
+        if 'tools' in config:
+            for tool_config in config['tools']:
+                _create_tool(
+                    specialist_id=specialist.id,
+                    tool_type=tool_config['type'],
+                    tool_version=tool_config['version'],
+                    name=tool_config.get('name'),
+                    description=tool_config.get('description'),
+                    timestamp=timestamp
+                )
+
+        db.session.commit()
+        current_app.logger.info(f"Successfully initialized crewai specialist {specialist.id}")
+
+    except SQLAlchemyError as e:
+        db.session.rollback()
+        current_app.logger.error(f"Database error initializing crewai specialist {specialist.id}: {str(e)}")
+        raise
+    except Exception as e:
+        db.session.rollback()
+        current_app.logger.error(f"Error initializing crewai specialist {specialist.id}: {str(e)}")
+        raise
+
+
+def _create_agent(
+        specialist_id: int,
+        agent_type: str,
+        agent_version: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        timestamp: Optional[dt] = None
+) -> EveAIAgent:
+    """Create an agent with the given configuration."""
+    if timestamp is None:
+        timestamp = dt.now(tz=tz.utc)
+
+    # Get agent configuration from cache
+    agent_config = cache_manager.agents_config_cache.get_config(agent_type, agent_version)
+
+    agent = EveAIAgent(
+        specialist_id=specialist_id,
+        name=name or agent_config.get('name', agent_type),
+        description=description or agent_config.get('metadata').get('description', ''),
+        type=agent_type,
+        type_version=agent_version,
+        role=None,
+        goal=None,
+        backstory=None,
+        tuning=False,
+        configuration=None,
+        arguments=None
+    )
+
+    set_logging_information(agent, timestamp)
+
+    db.session.add(agent)
+    current_app.logger.info(f"Created agent {agent.id} of type {agent_type}")
+    return agent
+
+
+def _create_task(
+        specialist_id: int,
+        task_type: str,
+        task_version: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        timestamp: Optional[dt] = None
+) -> EveAITask:
+    """Create a task with the given configuration."""
+    if timestamp is None:
+        timestamp = dt.now(tz=tz.utc)
+
+    # Get task configuration from cache
+    task_config = cache_manager.tasks_config_cache.get_config(task_type, task_version)
+
+    task = EveAITask(
+        specialist_id=specialist_id,
+        name=name or task_config.get('name', task_type),
+        description=description or task_config.get('metadata').get('description', ''),
+        type=task_type,
+        type_version=task_version,
+        task_description=None,
+        expected_output=None,
+        tuning=False,
+        configuration=None,
+        arguments=None,
+        context=None,
+        asynchronous=False,
+    )
+
+    set_logging_information(task, timestamp)
+
+    db.session.add(task)
+    current_app.logger.info(f"Created task {task.id} of type {task_type}")
+    return task
+
+
+def _create_tool(
+        specialist_id: int,
+        tool_type: str,
+        tool_version: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        timestamp: Optional[dt] = None
+) -> EveAITool:
+    """Create a tool with the given configuration."""
+    if timestamp is None:
+        timestamp = dt.now(tz=tz.utc)
+
+    # Get tool configuration from cache
+    tool_config = cache_manager.tools_config_cache.get_config(tool_type, tool_version)
+
+    tool = EveAITool(
+        specialist_id=specialist_id,
+        name=name or tool_config.get('name', tool_type),
+        description=description or tool_config.get('metadata').get('description', ''),
+        type=tool_type,
+        type_version=tool_version,
+        tuning=False,
+        configuration=None,
+        arguments=None,
+    )
+
+    set_logging_information(tool, timestamp)
+
+    db.session.add(tool)
+    current_app.logger.info(f"Created tool {tool.id} of type {tool_type}")
+    return tool

common/utils/startup_eveai.py

@@ -0,0 +1,47 @@
+import time
+
+from redis import Redis
+
+from common.extensions import cache_manager
+
+
+def perform_startup_actions(app):
+    perform_startup_invalidation(app)
+
+
+def perform_startup_invalidation(app):
+    """
+    Perform cache invalidation only once during startup using a persistent marker (also called flag or semaphore
+    - see docs).
+    Uses a combination of lock and marker to ensure invalidation happens exactly once
+    per deployment.
+    """
+    redis_client = Redis.from_url(app.config['REDIS_BASE_URI'])
+    startup_time = int(time.time())
+    marker_key = 'startup_invalidation_completed'
+    lock_key = 'startup_invalidation_lock'
+
+    try:
+        # First try to get the lock
+        lock = redis_client.lock(lock_key, timeout=30)
+        if lock.acquire(blocking=False):
+            try:
+                # Check if invalidation was already performed
+                if not redis_client.get(marker_key):
+                    # Perform invalidation
+                    cache_manager.invalidate_region('eveai_config')
+                    cache_manager.invalidate_region('eveai_chat_workers')
+
+                    redis_client.setex(marker_key, 180, str(startup_time))
+                    app.logger.info("Startup cache invalidation completed")
+                else:
+                    app.logger.info("Startup cache invalidation already performed")
+            finally:
+                lock.release()
+        else:
+            app.logger.info("Another process is handling startup invalidation")
+
+    except Exception as e:
+        app.logger.error(f"Error during startup invalidation: {e}")
+        # In case of error, we don't want to block the application startup
+        pass

config/agents/EMAIL_CONTENT_AGENT/1.0.0.yaml

@@ -0,0 +1,17 @@
+version: "1.0.0"
+name: "Email Content Agent"
+role: >
+  Email Content Writer
+goal: >
+  Craft a highly personalized email that resonates with the {end_user_role}'s context and identification (personal and 
+  company if available).
+  {custom_goal}
+backstory: >
+  You are an expert in writing compelling, personalized emails that capture the {end_user_role}'s attention and drive 
+  engagement. You are perfectly multilingual, and can write the mail in the native language of the {end_user_role}.
+  {custom_backstory}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that writes engaging emails."
+  changes: "Initial version"

config/agents/EMAIL_ENGAGEMENT_AGENT/1.0.0.yaml

@@ -0,0 +1,16 @@
+version: "1.0.0"
+name: "Email Engagement Agent"
+role: >
+  Engagement Optimization Specialist {custom_role}
+goal: >
+  You ensure that the email includes strong CTAs and strategically placed engagement hooks that encourage the 
+  {end_user_role} to take immediate action. {custom_goal}
+backstory: >
+  You specialize in optimizing content to ensure that it not only resonates with the recipient but also encourages them 
+  to take the desired action.
+  {custom_backstory}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that ensures the email is engaging and lead to maximal desired action"
+  changes: "Initial version"

config/agents/IDENTIFICATION_AGENT/1.0.0.yaml

@@ -0,0 +1,20 @@
+version: "1.0.0"
+name: "Identification Agent"
+role: >
+  Identification Administrative force. {custom_role}
+goal: >
+  You are an administrative force that tries to gather identification information to complete the administration of an 
+  end-user, the company he or she works for, through monitoring conversations and advising on questions to help you do
+  your job. You are responsible for completing the company's backend systems (like CRM, ERP, ...) with inputs from the 
+  end user in the conversation. 
+  {custom_goal}
+backstory: >
+  You are and administrative force for {company}, and very proficient in gathering information for the company's backend
+  systems. You do so by monitoring conversations between one of your colleagues (e.g. sales, finance, support, ...) and 
+  an end user. You ask your colleagues to request additional information to complete your task. 
+  {custom_backstory}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that gathers administrative information"
+  changes: "Initial version"

config/agents/RAG_AGENT/1.0.0.yaml

@@ -0,0 +1,22 @@
+version: "1.0.0"
+name: "Rag Agent"
+role: >
+  {company} Spokesperson. {custom_role}
+goal: >
+  You get questions by a human correspondent, and give answers based on a given context, taking into account the history
+  of the current conversation. {custom_goal}
+backstory: >
+  You are the primary contact for {company}. You are known by {name}, and can be addressed by this name, or you. You are
+  a very good communicator, and adapt to the style used by the human asking for information (e.g. formal or informal). 
+  You always stay correct and polite, whatever happens. And you ensure no discriminating language is used.
+  You are perfectly multilingual in all known languages, and do your best to answer questions in {language}, whatever
+  language the context provided to you is in. You are participating in a conversation, not writing e.g. an email. Do not
+  include a salutation or closing greeting in your answer.
+  {custom_backstory}
+full_model_name: "mistral.mistral-large-latest"
+temperature: 0.3
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that does RAG based on a user's question, RAG content & history"
+  changes: "Initial version"

config/agents/RAG_COMMUNICATION_AGENT/1.0.0.yaml

@@ -0,0 +1,26 @@
+version: "1.0.0"
+name: "Rag Communication Agent"
+role: >
+  {company} Interaction Responsible. {custom_role}
+goal: >
+  Your team has collected answers to a question asked. But it also created some additional questions to be asked. You
+  ensure the necessary answers are returned, and make an informed selection of the additional questions that can be 
+  asked (combining them when appropriate), ensuring the human you're communicating to does not get overwhelmed. 
+  {custom_goal}
+backstory: >
+  You are the online communication expert for {company}. You handled a lot of online communications with both customers
+  and internal employees. You are a master in redacting one coherent reply in a conversation that includes all the 
+  answers, and a selection of additional questions to be asked in a conversation. Although your backoffice team might 
+  want to ask a myriad of questions, you understand that doesn't fit with the way humans communicate. You know how to
+  combine multiple related questions, and understand how to interweave the questions in the answers when related. 
+  You are perfectly multilingual in all known languages, and do your best to answer questions in {language}, whatever
+  language the context provided to you is in. Also, ensure that questions asked do not contradict with the answers 
+  given, or aren't obsolete given the answer provided.
+  You are participating in a conversation, not writing e.g. an email. Do not include a salutation or closing greeting 
+  in your answer.
+  {custom_backstory}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that consolidates both answers and questions in a consistent reply"
+  changes: "Initial version"

config/agents/SPIN_DETECTION_AGENT/1.0.0.yaml

@@ -0,0 +1,22 @@
+version: "1.0.0"
+name: "SPIN Sales Assistant"
+role: >
+  Sales Assistant for {company} on {products}. {custom_role}
+goal: >
+  Your main job is to help your sales specialist to analyze an ongoing conversation with a customer, and detect 
+  SPIN-related information. {custom_goal}
+backstory: >
+  You are a sales assistant for {company}  on {products}. You are known by {name}, and can be addressed by this name, or you. You are
+  trained to understand an analyse ongoing conversations. Your are proficient in detecting SPIN-related information in a
+  conversation.
+  SPIN stands for:  
+  - Situation information - Understanding the customer's current context
+  - Problem information - Uncovering challenges and pain points
+  - Implication information - Exploring consequences of those problems
+  - Need-payoff information - Helping customers realize value of solutions
+  {custom_backstory}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that detects SPIN information in an ongoing conversation"
+  changes: "Initial version"

config/agents/SPIN_SALES_SPECIALIST_AGENT/1.0.0.yaml

@@ -0,0 +1,25 @@
+version: "1.0.0"
+name: "SPIN Sales Specialist"
+role: >
+  Sales Specialist for {company} on {products}. {custom_role}
+goal: >
+  Your main job is to do sales using the SPIN selling methodology in a first conversation with a potential customer. 
+  {custom_goal}
+backstory: >
+  You are a sales specialist for {company} on {products}. You are known by {name}, and can be addressed by this name, 
+  or you. You have an assistant that provides you with already detected SPIN-information in an ongoing conversation. You
+  decide on follow-up questions for more in-depth information to ensure we get the required information that may lead to
+  selling {products}. 
+  SPIN stands for:  
+  - Situation information - Understanding the customer's current context
+  - Problem information - Uncovering challenges and pain points
+  - Implication information - Exploring consequences of those problems
+  - Need-payoff information - Helping customers realize value of solutions
+  {custom_backstory}
+  You are acquainted with the following product information:
+  {product_information}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "An Agent that asks for Follow-up questions for SPIN-process"
+  changes: "Initial version"

config/assets/DOCUMENT_TEMPLATE/1.0.0.yaml

@@ -0,0 +1,18 @@
+version: "1.0.0"
+name: "Document Template"
+configuration:
+  argument_definition:
+    name: "variable_defition"
+    type: "file"
+    description: "Yaml file defining the arguments in the Document Template."
+    required: True
+  content_markdown:
+    name: "content_markdown"
+    type: "str"
+    description: "Actual template file in markdown format."
+    required: True
+metadata:
+  author: "Josako"
+  date_added: "2025-03-12"
+  description: "Asset that defines a template in markdown a specialist can process"
+  changes: "Initial version"

config/config.py

@@ -63,8 +63,10 @@ class Config(object):
     SUPPORTED_CURRENCIES = ['€', '$']
 
     # supported LLMs
-    SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
-    SUPPORTED_LLMS = ['openai.gpt-4o', 'anthropic.claude-3-5-sonnet', 'openai.gpt-4o-mini']
+    # SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
+    SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed']
+    SUPPORTED_LLMS = ['openai.gpt-4o', 'anthropic.claude-3-5-sonnet', 'openai.gpt-4o-mini',
+                      'mistral.mistral-large-latest', 'mistral.mistral-small-latest']
 
     ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', }
 
@@ -75,13 +77,10 @@ class Config(object):
         'anthropic.claude-3-5-sonnet': 8000
     }
 
-    # OpenAI API Keys
+    # Environemnt Loaders
     OPENAI_API_KEY = environ.get('OPENAI_API_KEY')
-
-    # Groq API Keys
+    MISTRAL_API_KEY = environ.get('MISTRAL_API_KEY')
     GROQ_API_KEY = environ.get('GROQ_API_KEY')
-
-    # Anthropic API Keys
     ANTHROPIC_API_KEY = environ.get('ANTHROPIC_API_KEY')
 
     # Celery settings
@@ -93,7 +92,7 @@ class Config(object):
 
     # SocketIO settings
     # SOCKETIO_ASYNC_MODE = 'threading'
-    SOCKETIO_ASYNC_MODE = 'gevent'
+    # SOCKETIO_ASYNC_MODE = 'gevent'
 
     # Session Settings
     SESSION_TYPE = 'redis'
@@ -105,6 +104,7 @@ class Config(object):
     # JWT settings
     JWT_SECRET_KEY = environ.get('JWT_SECRET_KEY')
     JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1)  # Set token expiry to 1 hour
+    JWT_ACCESS_TOKEN_EXPIRES_DEPLOY = timedelta(hours=24)   # Set long-lived token for deployment
 
     # API Encryption
     API_ENCRYPTION_KEY = environ.get('API_ENCRYPTION_KEY')
@@ -196,6 +196,8 @@ class DevConfig(Config):
     CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
     # eveai_chat_workers cache Redis Settings
     CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
+    # specialist execution pub/sub Redis Settings
+    SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
 
 
     # Unstructured settings
@@ -204,13 +206,13 @@ class DevConfig(Config):
     # UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general'
 
     # SocketIO settings
-    SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
-    SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
-    SOCKETIO_LOGGER = True
-    SOCKETIO_ENGINEIO_LOGGER = True
-    SOCKETIO_PING_TIMEOUT = 20000
-    SOCKETIO_PING_INTERVAL = 25000
-    SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
+    # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
+    # SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
+    # SOCKETIO_LOGGER = True
+    # SOCKETIO_ENGINEIO_LOGGER = True
+    # SOCKETIO_PING_TIMEOUT = 20000
+    # SOCKETIO_PING_INTERVAL = 25000
+    # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
                                                     # eveai-chat-widget.js
 
     # Google Cloud settings
@@ -289,18 +291,20 @@ class ProdConfig(Config):
     CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
     # eveai_chat_workers cache Redis Settings
     CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
+    # specialist execution pub/sub Redis Settings
+    SPECIALIST_EXEC_PUBSUB = f'{REDIS_BASE_URI}/5'
 
     # Session settings
     SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
 
     # SocketIO settings
-    SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
-    SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
-    SOCKETIO_LOGGER = True
-    SOCKETIO_ENGINEIO_LOGGER = True
-    SOCKETIO_PING_TIMEOUT = 20000
-    SOCKETIO_PING_INTERVAL = 25000
-    SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
+    # SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
+    # SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
+    # SOCKETIO_LOGGER = True
+    # SOCKETIO_ENGINEIO_LOGGER = True
+    # SOCKETIO_PING_TIMEOUT = 20000
+    # SOCKETIO_PING_INTERVAL = 25000
+    # SOCKETIO_MAX_IDLE_TIME = timedelta(minutes=60)  # Changing this value ==> change maxConnectionDuration value in
     # eveai-chat-widget.js
 
     # Google Cloud settings

config/prompts/encyclopedia/1.0.0.yaml

@@ -0,0 +1,12 @@
+version: "1.0.0"
+content: |
+  You have a lot of background knowledge, and as such you are some kind of 
+  'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question. 
+  If not, say you do not have sufficient information to answer the question. Use the {language} in your communication.
+  Question:
+  {question}
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "A background information retriever for Evie"
+  changes: "Initial version migrated from flat file structure"

config/prompts/history/1.0.0.yaml

@@ -0,0 +1,16 @@
+version: "1.0.0"
+content: |
+  You are a helpful assistant that details a question based on a conversation history, in such a way that the detailed 
+  question is understandable without that history. The conversation is a consequence of questions and context provided
+  by the HUMAN, and the AI (you) answering back, in chronological order. The most recent (i.e. last) elements are the
+  most important when detailing the question.
+  You answer by stating the detailed question in {language}.
+  History:
+  ```{history}```
+  Question to be detailed:
+  {question}
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "Prompt to further detail a question based on the previous conversation"
+  changes: "Initial version migrated from flat file structure"

config/prompts/html_parse/1.0.0.yaml

@@ -0,0 +1,21 @@
+version: "1.0.0"
+content: |
+  You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
+  
+  # Best practices are:
+  - Respect wordings and language(s) used in the HTML.
+  - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
+  - Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list.  
+  - Be careful of encoding of the text. Everything needs to be human readable.
+
+  Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text.
+
+  HTML is between triple backquotes.
+
+  ```{html}```
+model: "mistral.mistral-small-latest"
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "An aid in transforming HTML-based inputs to markdown"
+  changes: "Initial version migrated from flat file structure"

config/prompts/pdf_parse/1.0.0.yaml

@@ -0,0 +1,23 @@
+version: "1.0.0"
+content: |
+  You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
+  The content you get is already processed (some markdown already generated), but needs to be corrected. For large files, you may receive only portions of the full file. Consider this when processing the content.
+
+  # Best practices are:
+  - Respect wordings and language(s) used in the provided content.
+  - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
+  - When headings are numbered, show the numbering and define the header level. You may have to correct current header levels, as preprocessing is known to make errors.
+  - A new item is started when a <return> is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily).
+  - Paragraphs are to be stripped of newlines so they become easily readable.
+  - Be careful of encoding of the text. Everything needs to be human readable.
+
+  Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text.
+
+  PDF content is between triple backquotes.
+
+  ```{pdf_content}```
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "An assistant to parse PDF-content into markdown"
+  changes: "Initial version migrated from flat file structure"

config/prompts/rag/1.0.0.yaml

@@ -0,0 +1,15 @@
+version: "1.0.0"
+content: |
+  Answer the question based on the following context, delimited between triple backquotes. 
+  {tenant_context}
+  Use the following {language} in your communication, and cite the sources used at the end of the full conversation.
+  If the question cannot be answered using the given context, say "I have insufficient information to answer this question."
+  Context:
+  ```{context}```
+  Question:
+  {question}
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "The Main RAG retriever"
+  changes: "Initial version migrated from flat file structure"

config/prompts/summary/1.0.0.yaml

@@ -0,0 +1,9 @@
+version: "1.0.0"
+content: |
+  Write a concise summary of the text in {language}. The text is delimited between triple backquotes.
+  ```{text}```
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "An assistant to create a summary when multiple chunks are required for 1 file"
+  changes: "Initial version migrated from flat file structure"

config/prompts/transcript/1.0.0.yaml

@@ -0,0 +1,25 @@
+version: "1.0.0"
+content: |
+  You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
+  You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk.
+
+
+  # Best practices and steps are:
+  - Respect wordings and language(s) used in the transcription. Main language is {language}.
+  - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.    
+  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences.
+  - annotate the text to identify these logical parts using headings in {language}.
+  - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.
+
+  Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text.
+
+  The transcript is between triple backquotes.
+
+  $$${previous_part}$$$
+
+  ```{transcript}```
+metadata:
+  author: "Josako"
+  date_added: "2024-11-10"
+  description: "An assistant to transform a transcript to markdown."
+  changes: "Initial version migrated from flat file structure"

config/retrievers/DOSSIER_RETRIEVER/1.0.0.yaml

@@ -0,0 +1,36 @@
+version: "1.0.0"
+name: "DOSSIER Retriever"
+configuration:
+  es_k:
+    name: "es_k"
+    type: "int"
+    description: "K-value to retrieve embeddings (max embeddings retrieved)"
+    required: true
+    default: 8
+  es_similarity_threshold:
+    name: "es_similarity_threshold"
+    type: "float"
+    description: "Similarity threshold for retrieving embeddings"
+    required: true
+    default: 0.3
+  tagging_fields_filter:
+    name: "Tagging Fields Filter"
+    type: "tagging_fields_filter"
+    description: "Filter JSON to retrieve a subset of documents"
+    required: true
+  dynamic_arguments:
+    name: "Dynamic Arguments"
+    type: "dynamic_arguments"
+    description: "dynamic arguments used in the filter"
+    required: false
+arguments:
+  query:
+    name: "query"
+    type: "str"
+    description: "Query to retrieve embeddings"
+    required: True
+metadata:
+  author: "Josako"
+  date_added: "2025-03-11"
+  changes: "Initial version"
+  description: "Retrieving all embeddings conform the query and the tagging fields filter"

config/retrievers/STANDARD_RAG/1.0.0.yaml

@@ -0,0 +1,26 @@
+version: "1.0.0"
+name: "Standard RAG Retriever"
+configuration:
+  es_k:
+    name: "es_k"
+    type: "int"
+    description: "K-value to retrieve embeddings (max embeddings retrieved)"
+    required: true
+    default: 8
+  es_similarity_threshold:
+    name: "es_similarity_threshold"
+    type: "float"
+    description: "Similarity threshold for retrieving embeddings"
+    required: true
+    default: 0.3
+arguments:
+  query:
+    name: "query"
+    type: "str"
+    description: "Query to retrieve embeddings"
+    required: True
+metadata:
+  author: "Josako"
+  date_added: "2025-01-24"
+  changes: "Initial version"
+  description: "Retrieving all embeddings conform the query"

config/specialists/RAG_SPECIALIST/1.0.0.yaml

@@ -0,0 +1,53 @@
+version: "1.0.0"
+name: "RAG Specialist"
+framework: "crewai"
+configuration:
+  name:
+    name: "name"
+    type: "str"
+    description: "The name the specialist is called upon."
+    required: true
+  company:
+    name: "company"
+    type: "str"
+    description: "The name of your company. If not provided, your tenant's name will be used."
+    required: false
+arguments:
+  language:
+    name: "Language"
+    type: "str"
+    description: "Language code to be used for receiving questions and giving answers"
+    required: true
+  query:
+    name: "query"
+    type: "str"
+    description: "Query or response to process"
+    required: true
+results:
+  rag_output:
+    answer:
+      name: "answer"
+      type: "str"
+      description: "Answer to the query"
+      required: true
+    citations:
+      name: "citations"
+      type: "List[str]"
+      description: "List of citations"
+      required: false
+    insufficient_info:
+      name: "insufficient_info"
+      type: "bool"
+      description: "Whether or not the query is insufficient info"
+      required: true
+agents:
+  - type: "RAG_AGENT"
+    version: "1.0"
+tasks:
+  - type: "RAG_TASK"
+    version: "1.0"
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  changes: "Initial version"
+  description: "A Specialist that performs Q&A activities"

config/specialists/SPIN_SPECIALIST/1.0.0.yaml

@@ -0,0 +1,182 @@
+version: "1.0.0"
+name: "Spin Sales Specialist"
+framework: "crewai"
+configuration:
+  name:
+    name: "name"
+    type: "str"
+    description: "The name the specialist is called upon."
+    required: true
+  company:
+    name: "company"
+    type: "str"
+    description: "The name of your company. If not provided, your tenant's name will be used."
+    required: false
+  products:
+    name: "products"
+    type: "List[str]"
+    description: "The products or services you're providing"
+    required: false
+  product_information:
+    name: "product_information"
+    type: "text"
+    description: "Information on the products you are selling, such as ICP (Ideal Customer Profile), Pitch, ..."
+    required: false
+  engagement_options:
+    name: "engagement_options"
+    type: "text"
+    description: "Engagement options such as email, phone number, booking link, ..."
+  tenant_language:
+    name: "tenant_language"
+    type: "str"
+    description: "The language code used for internal information. If not provided, the tenant's default language will be used"
+    required: false
+  nr_of_questions:
+    name: "nr_of_questions"
+    type: "int"
+    description: "The maximum number of questions to formulate extra questions"
+    required: true
+    default: 3
+arguments:
+  language:
+    name: "Language"
+    type: "str"
+    description: "Language code to be used for receiving questions and giving answers"
+    required: true
+  query:
+    name: "query"
+    type: "str"
+    description: "Query or response to process"
+    required: true
+  identification:
+    name: "identification"
+    type: "text"
+    description: "Initial identification information when available"
+    required: false
+results:
+  rag_output:
+    answer:
+      name: "answer"
+      type: "str"
+      description: "Answer to the query"
+      required: true
+    citations:
+      name: "citations"
+      type: "List[str]"
+      description: "List of citations"
+      required: false
+    insufficient_info:
+      name: "insufficient_info"
+      type: "bool"
+      description: "Whether or not the query is insufficient info"
+      required: true
+  spin:
+    situation:
+      name: "situation"
+      type: "str"
+      description: "A description of the customer's current situation / context"
+      required: false
+    problem:
+      name: "problem"
+      type: "str"
+      description: "The current problems the customer is facing, for which he/she seeks a solution"
+      required: false
+    implication:
+      name: "implication"
+      type: "str"
+      description: "A list of implications"
+      required: false
+    needs:
+      name: "needs"
+      type: "str"
+      description: "A list of needs"
+      required: false
+    additional_info:
+      name: "additional_info"
+      type: "str"
+      description: "Additional information that may be commercially interesting"
+      required: false
+  lead_info:
+    lead_personal_info:
+      name:
+        name: "name"
+        type: "str"
+        description: "name of the lead"
+        required: "true"
+      job_title:
+        name: "job_title"
+        type: "str"
+        description: "job title"
+        required: false
+      email:
+        name: "email"
+        type: "str"
+        description: "lead email"
+        required: "false"
+      phone:
+        name: "phone"
+        type: "str"
+        description: "lead phone"
+        required: false
+      additional_info:
+        name: "additional_info"
+        type: "str"
+        description: "additional info on the lead"
+        required: false
+    lead_company_info:
+      company_name:
+        name: "company_name"
+        type: "str"
+        description: "Name of the lead company"
+        required: false
+      industry:
+        name: "industry"
+        type: "str"
+        description: "The industry of the company"
+        required: false
+      company_size:
+        name: "company_size"
+        type: "int"
+        description: "The size of the company"
+        required: false
+      company_website:
+        name: "company_website"
+        type: "str"
+        description: "The main website for the company"
+        required: false
+      additional_info:
+        name: "additional_info"
+        type: "str"
+        description: "Additional information that may be commercially interesting"
+        required: false
+agents:
+  - type: "RAG_AGENT"
+    version: "1.0"
+  - type: "RAG_COMMUNICATION_AGENT"
+    version: "1.0"
+  - type: "SPIN_DETECTION_AGENT"
+    version: "1.0"
+  - type: "SPIN_SALES_SPECIALIST_AGENT"
+    version: "1.0"
+  - type: "IDENTIFICATION_AGENT"
+    version: "1.0"
+  - type: "RAG_COMMUNICATION_AGENT"
+    version: "1.0"
+tasks:
+  - type: "RAG_TASK"
+    version: "1.0"
+  - type: "SPIN_DETECT_TASK"
+    version: "1.0"
+  - type: "SPIN_QUESTIONS_TASK"
+    version: "1.0"
+  - type: "IDENTIFICATION_DETECTION_TASK"
+    version: "1.0"
+  - type: "IDENTIFICATION_QUESTIONS_TASK"
+    version: "1.0"
+  - type: "RAG_CONSOLIDATION_TASK"
+    version: "1.0"
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  changes: "Initial version"
+  description: "A Specialist that performs both Q&A as SPIN (Sales Process) activities"

config/specialists/STANDARD_RAG_SPECIALIST/1.0.0.yaml

@@ -0,0 +1,52 @@
+version: 1.0.0
+name: "Standard RAG Specialist"
+framework: "langchain"
+configuration:
+  specialist_context:
+    name: "Specialist Context"
+    type: "text"
+    description: "The context to be used by the specialist."
+    required: false
+  temperature:
+    name: "Temperature"
+    type: "number"
+    description: "The inference temperature to be used by the specialist."
+    required: false
+    default: 0.3
+arguments:
+  language:
+    name: "Language"
+    type: "str"
+    description: "Language code to be used for receiving questions and giving answers"
+    required: true
+  query:
+    name: "query"
+    type: "str"
+    description: "Query to answer"
+    required: true
+results:
+  detailed_query:
+    name: "detailed_query"
+    type: "str"
+    description: "The query detailed with the Chat Session History."
+    required: true
+  answer:
+    name: "answer"
+    type: "str"
+    description: "Answer to the query"
+    required: true
+  citations:
+    name: "citations"
+    type: "List[str]"
+    description: "List of citations"
+    required: false
+  insufficient_info:
+    name: "insufficient_info"
+    type: "bool"
+    description: "Whether or not the query is insufficient info"
+    required: true
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  changes: "Initial version"
+  description: "A Specialist that performs standard Q&A"

config/tasks/EMAIL_LEAD_DRAFTING_TASK/1.0.0.yaml

@@ -0,0 +1,35 @@
+version: "1.0.0"
+name: "Email Lead Draft Creation"
+task_description: >
+  Craft a highly personalized email using the lead's name, job title, company information, and any relevant personal or 
+  company achievements when available. The email should speak directly to the lead's interests and the needs
+  of their company.
+  This mail is the consequence of a first conversation. You have information available from that conversation in the 
+    - SPIN-context (in between triple %)
+    - personal and company information (in between triple $)
+  Information might be missing however, as it might not be gathered in that first conversation.
+  Don't use any salutations or closing remarks, nor too complex sentences.
+
+  Our Company and Product:
+  - Company Name: {company}
+  - Products: {products}
+  - Product information: {product_information}
+
+  {customer_role}'s Identification:
+  $$${Identification}$$$
+  
+  SPIN context:
+  %%%{SPIN}%%%
+  
+  {custom_description}
+expected_output: >
+  A personalized email draft that:
+    - Addresses the lead by name
+    - Acknowledges their role and company
+    - Highlights how {company} can meet their specific needs or interests
+  {customer_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "Email Drafting Task towards a Lead"
+  changes: "Initial version"

config/tasks/EMAIL_LEAD_ENGAGEMENT_TASK/1.0.0.yaml

@@ -0,0 +1,28 @@
+version: "1.0.0"
+name: "Email Lead Engagement Creation"
+task_description: >
+  Review a personalized email and optimize it with strong CTAs and engagement hooks. Keep in mind that this email is
+  the consequence of a first conversation. 
+  Don't use any salutations or closing remarks, nor too complex sentences. Keep it short and to the point.
+  Don't use any salutations or closing remarks, nor too complex sentences.
+  Ensure the email encourages the lead to schedule a meeting or take
+  another desired action immediately.
+
+  Our Company and Product:
+    - Company Name: {company}
+    - Products: {products}
+    - Product information: {product_information}
+
+  Engagement options:
+  {engagement_options}
+  
+  {custom_description}
+expected_output: >
+  An optimized email ready for sending, complete with:
+    - Strong CTAs
+    - Strategically placed engagement hooks that encourage immediate action
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "Make an Email draft more engaging"
+  changes: "Initial version"

config/tasks/IDENTIFICATION_DETECTION_TASK/1.0.0.yaml

@@ -0,0 +1,27 @@
+version: "1.0.0"
+name: "Identification Gathering"
+task_description: >
+  You are asked to gather lead information in a conversation with a new prospect. This is information about the person
+  participating in the conversation, and information on the company he or she is working for. Try to be as precise as
+  possible.
+  Take into account information already gathered in the historic lead info (between triple backquotes) and add 
+  information found in the latest reply. Also, some identification information may be given by the end user.
+  
+  historic lead info:
+    ```{historic_lead_info}```
+  latest reply:
+    {query}
+  identification:
+    {identification}
+  
+  {custom_description}
+expected_output: >
+  - Personal Identification information such as name, email, phone number, job title, and any additional information that
+  may prove to be interesting in the current or future conversations.
+  - Company information such as company name, industry, size, company website, ...
+  {custom_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "A Task that gathers identification information from a conversation"
+  changes: "Initial version"

config/tasks/IDENTIFICATION_QUESTIONS_TASK/1.0.0.yaml

@@ -0,0 +1,24 @@
+version: "1.0.0"
+name: "Define Identification Questions"
+task_description: >
+  Gather the identification information gathered by your team mates. Ensure no information in the historic lead 
+  information (in between triple backquotes) and the latest reply of the user is lost. 
+  Define questions to be asked to complete the personal and company information for the end user in the conversation.
+  historic lead info:
+    ```{historic_lead_info}```
+  latest reply:
+    {query}
+  
+  {custom_description}
+expected_output: >
+  - Personal Identification information such as name, email, phone number, job title, and any additional information that
+    may prove to be interesting in the current or future conversations.
+  - Company information such as company name, industry, size, company website, ...
+    {custom_expected_output}
+  - Top {nr_of_questions} questions to ask in order to complete identification.
+  {custom_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "A Task to define identification (person & company) questions"
+  changes: "Initial version"

config/tasks/RAG_CONSOLIDATION_TASK/1.0.0.yaml

@@ -0,0 +1,28 @@
+version: "1.0.0"
+name: "Rag Consolidation"
+task_description: >
+  Your teams have collected answers to a user's query (in between triple backquotes), and collected additional follow-up
+  questions (in between triple %) to reach their goals. Ensure the answers are provided, and select a maximum of 
+  {nr_of_questions} out of the additional questions to be asked in order not to overwhelm the user. The questions are 
+  in no specific order, so don't just pick the first ones. Make a good mixture of different types of questions, 
+  different topics or subjects!  
+  Questions are to be asked when your team proposes questions. You ensure both answers and additional questions are 
+  bundled into 1 clear communication back to the user. Use {language} for your consolidated communication.
+  Be sure to format your answer in markdown when appropriate. Ensure enumerations or bulleted lists are formatted as
+  lists in markdown.
+  {custom_description}
+  
+  Anwers:
+  ```{prepared_answers}```
+  
+  Additional Questions:
+  %%%{additional_questions}%%%
+
+expected_output: >
+  One consolidated communication towards the end user.
+  {custom_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "A Task to consolidate questions and answers"
+  changes: "Initial version"

config/tasks/RAG_TASK/1.0.0.yaml

@@ -0,0 +1,24 @@
+version: "1.0.0"
+name: "RAG Task"
+task_description: >
+  Answer the query based on the following context, delimited between triple backquotes, and taking into account
+  the history of the discussion, in between triple %. Try not to repeat answers already given in the recent history, 
+  unless confirmation is required or repetition is essential to give a coherent answer.
+  {custom_description}
+  Use the following {language} in your communication, and cite the sources used at the end of the full conversation.
+  If the question cannot be answered using the given context, answer "I have insufficient information to answer this question."
+  Context:
+    ```{context}```
+  History:
+    %%%{history}%%%
+  Query:
+    {query}
+expected_output: >
+  - Answer
+  - A list of sources used in generating the answer, citations
+  - An indication (True or False) if there's insufficient information to give an answer.
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "A Task that gives RAG-based answers"
+  changes: "Initial version"

config/tasks/SPIN_DETECT_TASK/1.0.0.yaml

@@ -0,0 +1,24 @@
+version: "1.0.0"
+name: "SPIN Information Detection"
+task_description: >
+  Complement the historic SPIN context (in between triple backquotes) with information found in the latest reply of the
+  end user. 
+  {custom_description}
+  Use the following {tenant_language} to define the SPIN-elements. 
+  Historic SPIN:
+    ```{historic_spin}```
+  Latest reply:
+    {query}
+expected_output: >
+  The SPIN analysis, comprised of:
+  - Situation information: a description of the customer's current context / situation.
+  - Problem information: a description of the customer's problems uncovering it's challenges and pain points.
+  - Implication information: implications of situation / identified problems, i.e. of the consequences of those problems.
+  - Need-payoff information: Customer's needs, helping customers realize value of solutions.
+  - Additional info: Information that does not fit in the above SPIN-categories, but that can be commercially 
+  interesting, including if provided: {custom_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "A Task that performs SPIN Information Detection"
+  changes: "Initial version"

config/tasks/SPIN_QUESTIONS_TASK/1.0.0.yaml

@@ -0,0 +1,30 @@
+version: "1.0.0"
+name: "SPIN Question Identification"
+task_description: >
+  Revise the final SPIN provided by your colleague, and ensure no information is lost from the histoic SPIN and the 
+  latest reply from the user. Define the top questions that need to be asked to understand the full SPIN context
+  of the customer. If you think this user could be a potential customer, please indicate so.
+  {custom_description}
+  Use the following {tenant_language} to define the SPIN-elements. If you have a satisfying SPIN context, just skip and 
+  don't ask for more information or confirmation.
+  Historic SPIN:
+      ```{historic_spin}```
+  Latest reply:
+    {query}
+expected_output: >
+  The SPIN analysis, comprised of:
+   - Situation information: a description of the customer's current context / situation.
+   - Problem information: a description of the customer's problems uncovering it's challenges and pain points.
+   - Implication information: implications of situation / identified problems, i.e. of the consequences of those problems.
+   - Need-payoff information: Customer's needs, helping customers realize value of solutions.
+   - Additional info: Information that does not fit in the above SPIN-categories, but that can be commercially interesting.
+  The SPIN questions:
+  - At max {nr_of_questions} questions to complete the SPIN-context of the customer, as a markdown list without '```'.
+  Potential Customer Indication:
+  - An indication if - given the current SPIN - this could be a good customer (True) or not (False).
+  {custom_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-08"
+  description: "A Task that identifies questions to complete the SPIN context in a conversation"
+  changes: "Initial version"

config/type_defs/agent_types.py

@@ -0,0 +1,31 @@
+# Agent Types
+AGENT_TYPES = {
+    "EMAIL_CONTENT_AGENT": {
+        "name": "Email Content Agent",
+        "description": "An Agent that writes engaging emails.",
+    },
+    "EMAIL_ENGAGEMENT_AGENT": {
+        "name": "Email Engagement Agent",
+        "description": "An Agent that ensures the email is engaging and lead to maximal desired action",
+    },
+    "IDENTIFICATION_AGENT": {
+        "name": "Identification Agent",
+        "description": "An Agent that gathers identification information",
+    },
+    "RAG_AGENT": {
+        "name": "Rag Agent",
+        "description": "An Agent that does RAG based on a user's question, RAG content & history",
+    },
+    "RAG_COMMUNICATION_AGENT": {
+        "name": "Rag Communication Agent",
+        "description": "An Agent that consolidates both answers and questions in a consistent reply",
+    },
+    "SPIN_DETECTION_AGENT": {
+        "name": "SPIN Sales Assistant",
+        "description": "An Agent that detects SPIN information in an ongoing conversation",
+    },
+    "SPIN_SALES_SPECIALIST_AGENT": {
+        "name": "SPIN Sales Specialist",
+        "description": "An Agent that asks for Follow-up questions for SPIN-process",
+    },
+}

config/type_defs/asset_types.py

@@ -0,0 +1,7 @@
+# Agent Types
+AGENT_TYPES = {
+    "DOCUMENT_TEMPLATE": {
+        "name": "Document Template",
+        "description": "Asset that defines a template in markdown a specialist can process",
+    },
+}

config/type_defs/catalog_types.py

@@ -6,7 +6,7 @@ CATALOG_TYPES = {
         "configuration": {},
         "document_version_configurations": []
     },
-    "DOSSIER": {
+    "DOSSIER_CATALOG": {
         "name": "Dossier Catalog",
         "Description": "A Catalog with information in Evie's Library in which several Dossiers can be stored",
         "configuration": {

config/type_defs/prompt_types.py

@@ -0,0 +1,31 @@
+# Agent Types
+PROMPT_TYPES = {
+    "encyclopedia": {
+        "name": "encyclopedia",
+        "description": "A background information retriever for Evie",
+    },
+    "history": {
+        "name": "history",
+        "description": "Prompt to further detail a question based on the previous conversation",
+    },
+    "html_parse": {
+        "name": "html_parse",
+        "description": "An aid in transforming HTML-based inputs to markdown",
+    },
+    "pdf_parse": {
+        "name": "pdf_parse",
+        "description": "An assistant to parse PDF-content into markdown",
+    },
+    "rag": {
+        "name": "rag",
+        "description": "The Main RAG retriever",
+    },
+    "summary": {
+        "name": "summary",
+        "description": "An assistant to create a summary when multiple chunks are required for 1 file",
+    },
+    "transcript": {
+        "name": "transcript",
+        "description": "An assistant to transform a transcript to markdown.",
+    },
+}

config/type_defs/retriever_types.py

@@ -2,30 +2,10 @@
 RETRIEVER_TYPES = {
     "STANDARD_RAG": {
         "name": "Standard RAG Retriever",
-        "description": "Retrieving all embeddings conform the query",
-        "configuration": {
-            "es_k": {
-                "name": "es_k",
-                "type": "int",
-                "description": "K-value to retrieve embeddings (max embeddings retrieved)",
-                "required": True,
-                "default": 8,
-            },
-            "es_similarity_threshold": {
-                "name": "es_similarity_threshold",
-                "type": "float",
-                "description": "Similarity threshold for retrieving embeddings",
-                "required": True,
-                "default": 0.3,
-            },
-        },
-        "arguments": {
-            "query": {
-                "name": "query",
-                "type": "str",
-                "description": "Query to retrieve embeddings",
-                "required": True,
-            },
-        }
+        "description": "Retrieving all embeddings from the catalog conform the query",
+    },
+    "DOSSIER_RETRIEVER": {
+        "name": "Retriever for managing DOSSIER catalogs",
+        "description": "Retrieving filtered embeddings from the catalog conform the query",
     }
 }

config/type_defs/service_types.py

@@ -8,4 +8,12 @@ SERVICE_TYPES = {
         "name": "DOCAPI",
         "description": "Service allows to use document API functionality.",
     },
+    "DEPLOY_API": {
+        "name": "DEPLOY_API",
+        "description": "Service allows to use deployment API functionality.",
+    },
+    "SPECIALIST_API": {
+        "name": "SPECIALIST_API",
+        "description": "Service allows to use specialist execution API functionality.",
+    }
 }

config/type_defs/specialist_types.py

@@ -1,62 +1,15 @@
 # Specialist Types
 SPECIALIST_TYPES = {
-    "STANDARD_RAG": {
+    "STANDARD_RAG_SPECIALIST": {
         "name": "Q&A RAG Specialist",
         "description": "Standard Q&A through RAG Specialist",
-        "configuration": {
-            "specialist_context": {
-                "name": "Specialist Context",
-                "type": "text",
-                "description": "The context to be used by the specialist.",
-                "required": False,
-            },
-            "temperature": {
-                "name": "Temperature",
-                "type": "number",
-                "description": "The inference temperature to be used by the specialist.",
-                "required": False,
-                "default": 0.3
-            }
-        },
-        "arguments": {
-            "language": {
-                "name": "Language",
-                "type": "str",
-                "description": "Language code to be used for receiving questions and giving answers",
-                "required": True,
-            },
-            "query": {
-                "name": "query",
-                "type": "str",
-                "description": "Query to answer",
-                "required": True,
-            }
-        },
-        "results": {
-            "detailed_query": {
-                "name": "detailed_query",
-                "type": "str",
-                "description": "The query detailed with the Chat Session History.",
-                "required": True,
-            },
-            "answer": {
-                "name": "answer",
-                "type": "str",
-                "description": "Answer to the query",
-                "required": True,
-            },
-            "citations": {
-                "name": "citations",
-                "type": "List[str]",
-                "description": "List of citations",
-                "required": False,
-            },
-            "insufficient_info": {
-                "name": "insufficient_info",
-                "type": "bool",
-                "description": "Whether or not the query is insufficient info",
-                "required": True,
-            },
-        }
+    },
+    "RAG_SPECIALIST": {
+        "name": "RAG Specialist",
+        "description": "Q&A through RAG Specialist",
+    },
+    "SPIN_SPECIALIST": {
+        "name": "Spin Sales Specialist",
+        "description": "A specialist that allows to answer user queries, try to get SPIN-information and Identification",
     }
 }

config/type_defs/task_types.py

@@ -0,0 +1,35 @@
+# Agent Types
+TASK_TYPES = {
+    "EMAIL_LEAD_DRAFTING_TASK": {
+        "name": "Email Lead Draft Creation",
+        "description": "Email Drafting Task towards a Lead",
+    },
+    "EMAIL_LEAD_ENGAGEMENT_TASK": {
+        "name": "Email Lead Engagement Creation",
+        "description": "Make an Email draft more engaging",
+    },
+    "IDENTIFICATION_DETECTION_TASK": {
+        "name": "Identification Gathering",
+        "description": "A Task that gathers identification information from a conversation",
+    },
+    "IDENTIFICATION_QUESTIONS_TASK": {
+        "name": "Define Identification Questions",
+        "description": "A Task to define identification (person & company) questions",
+    },
+    "RAG_TASK": {
+        "name": "RAG Task",
+        "description": "A Task that gives RAG-based answers",
+    },
+    "SPIN_DETECT_TASK": {
+        "name": "SPIN Information Detection",
+        "description": "A Task that performs SPIN Information Detection",
+    },
+    "SPIN_QUESTIONS_TASK": {
+        "name": "SPIN Question Identification",
+        "description": "A Task that identifies questions to complete the SPIN context in a conversation",
+    },
+    "RAG_CONSOLIDATION_TASK": {
+        "name": "RAG Consolidation",
+        "description": "A Task to consolidate questions and answers",
+    }
+}

config/type_defs/tool_types.py

@@ -0,0 +1,4 @@
+# Agent Types
+TOOL_TYPES = {
+
+}

docker/build_and_push_eveai.sh

@@ -158,6 +158,9 @@ docker buildx use eveai_builder
 
 # Loop through services
 for SERVICE in "${SERVICES[@]}"; do
+    if [[ "$SERVICE" == "nginx" ]]; then
+      ./copy_specialist_svgs.sh ../config ../nginx/static/assets
+    fi
     if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* || "$SERVICE" == "flower" ]]; then
         if process_service "$SERVICE"; then
             echo "Successfully processed $SERVICE"

docker/compose_dev.yaml

@@ -28,6 +28,7 @@ x-common-variables: &common-variables
   FLOWER_PASSWORD: 'Jungles'
   OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7'
   GROQ_API_KEY: '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'
+  MISTRAL_API_KEY: 'jGDc6fkCbt0iOC0jQsbuZhcjLWBPGc2b'
   ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2'
   JWT_SECRET_KEY: 'bsdMkmQ8ObfMD52yAFg4trrvjgjMhuIqg2fjDpD/JqvgY0ccCcmlsEnVFmR79WPiLKEA3i8a5zmejwLZKl4v9Q=='
   API_ENCRYPTION_KEY: 'xfF5369IsredSrlrYZqkM9ZNrfUASYYS6TCcAR9UKj4='
@@ -36,6 +37,8 @@ x-common-variables: &common-variables
   MINIO_SECRET_KEY: minioadmin
   NGINX_SERVER_NAME: 'localhost http://macstudio.ask-eve-ai-local.com/'
   LANGCHAIN_API_KEY: "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"
+  SERPER_API_KEY: "e4c553856d0e6b5a171ec5e6b69d874285b9badf"
+  CREWAI_STORAGE_DIR: "/app/crewai_storage"
 
 services:
   nginx:
@@ -63,7 +66,7 @@ services:
       - ./logs/nginx:/var/log/nginx
     depends_on:
       - eveai_app
-      - eveai_chat
+      - eveai_api
     networks:
       - eveai-network
 
@@ -77,6 +80,8 @@ services:
         - linux/arm64
     ports:
       - 5001:5001
+    expose:
+      - 8000
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_app
@@ -112,6 +117,8 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
+    expose:
+      - 8000
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_workers
@@ -132,39 +139,39 @@ services:
     networks:
       - eveai-network
 
-  eveai_chat:
-    image: josakola/eveai_chat:latest
-    build:
-      context: ..
-      dockerfile: ./docker/eveai_chat/Dockerfile
-      platforms:
-        - linux/amd64
-        - linux/arm64
-    ports:
-      - 5002:5002
-    environment:
-      <<: *common-variables
-      COMPONENT_NAME: eveai_chat
-    volumes:
-      - ../eveai_chat:/app/eveai_chat
-      - ../common:/app/common
-      - ../config:/app/config
-      - ../scripts:/app/scripts
-      - ../patched_packages:/app/patched_packages
-      - ./eveai_logs:/app/logs
-    depends_on:
-      db:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
-      interval: 30s
-      timeout: 1s
-      retries: 3
-      start_period: 30s
-    networks:
-      - eveai-network
+#  eveai_chat:
+#    image: josakola/eveai_chat:latest
+#    build:
+#      context: ..
+#      dockerfile: ./docker/eveai_chat/Dockerfile
+#      platforms:
+#        - linux/amd64
+#        - linux/arm64
+#    ports:
+#      - 5002:5002
+#    environment:
+#      <<: *common-variables
+#      COMPONENT_NAME: eveai_chat
+#    volumes:
+#      - ../eveai_chat:/app/eveai_chat
+#      - ../common:/app/common
+#      - ../config:/app/config
+#      - ../scripts:/app/scripts
+#      - ../patched_packages:/app/patched_packages
+#      - ./eveai_logs:/app/logs
+#    depends_on:
+#      db:
+#        condition: service_healthy
+#      redis:
+#        condition: service_healthy
+#    healthcheck:
+#      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
+#      interval: 30s
+#      timeout: 1s
+#      retries: 3
+#      start_period: 30s
+#    networks:
+#      - eveai-network
 
   eveai_chat_workers:
     image: josakola/eveai_chat_workers:latest
@@ -174,6 +181,8 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
+    expose:
+      - 8000
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_chat_workers
@@ -202,6 +211,8 @@ services:
         - linux/arm64
     ports:
       - 5003:5003
+    expose:
+      - 8000
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_api
@@ -263,6 +274,8 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
+    expose:
+      - 8000
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_entitlements
@@ -358,6 +371,42 @@ services:
     networks:
       - eveai-network
 
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus/data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    restart: unless-stopped
+    networks:
+      - eveai-network
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    restart: unless-stopped
+    depends_on:
+      - prometheus
+    networks:
+      - eveai-network
+
 networks:
   eveai-network:
     driver: bridge

docker/compose_stackhero.yaml

@@ -31,6 +31,7 @@ x-common-variables: &common-variables
   OPENAI_API_KEY: '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'
   GROQ_API_KEY: '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'
   ANTHROPIC_API_KEY: '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'
+  MISTRAL_API_KEY: 'PjnUeDRPD7B144wdHlH0CzR7m0z8RHXi'
   JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408'
   API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0='
   GRAYLOG_HOST: de4zvu.stackhero-network.com
@@ -40,6 +41,8 @@ x-common-variables: &common-variables
   MINIO_SECRET_KEY: 2PEZAD1nlpAmOyDV0TUTuJTQw1qVuYLF3A7GMs0D
   NGINX_SERVER_NAME: 'evie.askeveai.com mxz536.stackhero-network.com'
   LANGCHAIN_API_KEY: "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"
+  SERPER_API_KEY: "e4c553856d0e6b5a171ec5e6b69d874285b9badf"
+  CREWAI_STORAGE_DIR: "/app/crewai_storage"
 
 networks:
   eveai-network:
@@ -64,7 +67,7 @@ services:
       - "traefik.http.services.nginx.loadbalancer.server.port=80"
     depends_on:
       - eveai_app
-      - eveai_chat
+      - eveai_api
     networks:
       - eveai-network
 
@@ -97,23 +100,23 @@ services:
     networks:
       - eveai-network
 
-  eveai_chat:
-    platform: linux/amd64
-    image: josakola/eveai_chat:latest
-    ports:
-      - 5002:5002
-    environment:
-      <<: *common-variables
-      COMPONENT_NAME: eveai_chat
-    volumes:
-      - eveai_logs:/app/logs
-    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    networks:
-      - eveai-network
+#  eveai_chat:
+#    platform: linux/amd64
+#    image: josakola/eveai_chat:latest
+#    ports:
+#      - 5002:5002
+#    environment:
+#      <<: *common-variables
+#      COMPONENT_NAME: eveai_chat
+#    volumes:
+#      - eveai_logs:/app/logs
+#    healthcheck:
+#      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
+#      interval: 10s
+#      timeout: 5s
+#      retries: 5
+#    networks:
+#      - eveai-network
 
   eveai_chat_workers:
     platform: linux/amd64

docker/copy_specialist_svgs.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Script to copy specialist overview SVGs to nginx static directory
+
+# Function to show usage
+show_usage() {
+    echo "Usage: $0 <config_dir> <static_dir>"
+    echo "  config_dir: Path to the config directory containing specialists"
+    echo "  static_dir: Path to the nginx static directory"
+    exit 1
+}
+
+# Check arguments
+if [ $# -ne 2 ]; then
+    show_usage
+fi
+
+CONFIG_DIR="$1"
+STATIC_DIR="$2"
+SPECIALISTS_DIR="${CONFIG_DIR}/specialists"
+OUTPUT_DIR="${STATIC_DIR}/specialists"
+
+# Check if source directory exists
+if [ ! -d "$SPECIALISTS_DIR" ]; then
+    echo "Error: Specialists directory not found at $SPECIALISTS_DIR"
+    exit 1
+fi
+
+# Create output directory
+mkdir -p "$OUTPUT_DIR"
+
+# Counter for processed files
+processed=0
+
+# Process each specialist type directory
+for TYPE_DIR in "$SPECIALISTS_DIR"/*/ ; do
+    if [ -d "$TYPE_DIR" ]; then
+        # Get specialist type from directory name
+        SPECIALIST_TYPE=$(basename "$TYPE_DIR")
+
+        # Find and process overview SVG files
+        for SVG_FILE in "$TYPE_DIR"*_overview.svg; do
+            if [ -f "$SVG_FILE" ]; then
+                # Extract version (remove _overview.svg from filename)
+                VERSION=$(basename "$SVG_FILE" "_overview.svg")
+
+                # Create new filename
+                NEW_FILENAME="${SPECIALIST_TYPE}_${VERSION}_overview.svg"
+
+                # Copy file
+                cp -f "$SVG_FILE" "${OUTPUT_DIR}/${NEW_FILENAME}"
+
+                echo "Copied $(basename "$SVG_FILE") -> $NEW_FILENAME"
+                ((processed++))
+            fi
+        done
+    fi
+done
+
+echo -e "\nProcessed $processed overview SVG files"

docker/eveai_chat_workers/Dockerfile

@@ -39,6 +39,7 @@ RUN apt-get update && apt-get install -y \
 
 # Create logs directory and set permissions
 RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+RUN mkdir -p /app/crewai_storage && chown -R appuser:appuser /app/crewai_storage
 
 # Download dependencies as a separate step to take advantage of Docker's caching.
 # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.

docker/grafana/provisioning/dashboards/eveai-system-dashboard.json

@@ -0,0 +1,627 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "color": {
+            "fixedColor": "#76599a",
+            "mode": "fixed"
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(eveai_business_events_total[$__interval])) by (event_type)",
+          "refId": "A"
+        }
+      ],
+      "title": "Business Events by Type",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 5
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.5.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(eveai_business_events_concurrent)",
+          "refId": "A"
+        }
+      ],
+      "title": "Concurrent Business Events",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.5.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(eveai_business_events_total[$__range])) by (specialist_type)",
+          "refId": "A"
+        }
+      ],
+      "title": "Events by Specialist Type",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "histogram_quantile(0.95, sum(rate(eveai_business_events_duration_seconds_bucket[$__interval])) by (le, event_type))",
+          "refId": "A"
+        }
+      ],
+      "title": "Business Event Duration (95th percentile)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(eveai_business_spans_total[$__interval])) by (activity_name)",
+          "refId": "A"
+        }
+      ],
+      "title": "Activity Execution Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 60,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 0,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(eveai_llm_tokens_total[$__interval])) by (token_type)",
+          "refId": "A"
+        }
+      ],
+      "title": "LLM Token Usage by Type",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "histogram_quantile(0.95, sum(rate(eveai_llm_duration_seconds_bucket[$__interval])) by (le, interaction_type))",
+          "refId": "A"
+        }
+      ],
+      "title": "LLM Duration by Interaction Type (95th percentile)",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "15m",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": ["eveai", "system"],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "PBFA97CFB590B2093"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "EveAI System Dashboard",
+  "uid": "eveai-system-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

docker/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true

docker/nginx/Dockerfile

@@ -9,11 +9,14 @@ COPY ../../nginx/mime.types /etc/nginx/mime.types
 
 # Copy static & public files
 RUN mkdir -p /etc/nginx/static /etc/nginx/public
+
 COPY ../../nginx/static /etc/nginx/static
 COPY ../../integrations/Wordpress/eveai-chat/assets/css/eveai-chat-style.css /etc/nginx/static/css/
 COPY ../../integrations/Wordpress/eveai-chat/assets/js/eveai-chat-widget.js /etc/nginx/static/js/
 COPY ../../integrations/Wordpress/eveai-chat/assets/js/eveai-token-manager.js /etc/nginx/static/js/
 COPY ../../integrations/Wordpress/eveai-chat/assets/js/eveai-sdk.js /etc/nginx/static/js
+
+# Copy public files
 COPY ../../nginx/public /etc/nginx/public
 
 # Copy site-specific configurations

docker/prometheus/prometheus.yml

@@ -0,0 +1,34 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  scrape_timeout: 10s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'eveai_app'
+    static_configs:
+      - targets: ['eveai_app:8000']
+    scrape_interval: 10s
+
+  - job_name: 'eveai_workers'
+    static_configs:
+      - targets: ['eveai_workers:8000']
+    scrape_interval: 10s
+
+  - job_name: 'eveai_chat_workers'
+    static_configs:
+      - targets: ['eveai_chat_workers:8000']
+    scrape_interval: 10s
+
+  - job_name: 'eveai_api'
+    static_configs:
+      - targets: ['eveai_api:8000']
+    scrape_interval: 10s
+
+  - job_name: 'eveai_entitlements'
+    static_configs:
+      - targets: ['eveai_entitlements:8000']
+    scrape_interval: 10s

eveai_api/__init__.py

@@ -1,11 +1,11 @@
 import traceback
 
-from flask import Flask, jsonify, request
+from flask import Flask, jsonify, request, redirect
 from flask_jwt_extended import get_jwt_identity, verify_jwt_in_request
 from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.exceptions import HTTPException
 
-from common.extensions import db, api_rest, jwt, minio_client, simple_encryption, cors
+from common.extensions import db, api_rest, jwt, minio_client, simple_encryption, cors, cache_manager
 import os
 import logging.config
 
@@ -15,6 +15,7 @@ from common.utils.database import Database
 from config.logging_config import LOGGING
 from .api.document_api import document_ns
 from .api.auth import auth_ns
+from .api.specialist_execution_api import specialist_execution_ns
 from config.config import get_config
 from common.utils.celery_utils import make_celery, init_celery
 from common.utils.eveai_exceptions import EveAIException
@@ -30,7 +31,7 @@ def create_app(config_file=None):
         case 'development':
             app.config.from_object(get_config('dev'))
         case 'production':
-            app.config.from_object(get_config('prod'))  
+            app.config.from_object(get_config('prod'))
         case _:
             app.config.from_object(get_config('dev'))
 
@@ -59,10 +60,12 @@ def create_app(config_file=None):
     # Register Request Debugger
     register_request_debugger(app)
 
+    # Register Cache Handlers
+    register_cache_handlers(app)
+
     @app.before_request
     def check_cors():
         if request.method == 'OPTIONS':
-            app.logger.debug("Handling OPTIONS request")
             return '', 200  # Allow OPTIONS to pass through
 
         origin = request.headers.get('Origin')
@@ -103,17 +106,23 @@ def create_app(config_file=None):
 
     @app.route('/api/v1')
     def swagger():
-        return api_rest.render_doc()
+        return redirect('/api/v1/')
 
     return app
 
 
 def register_extensions(app):
     db.init_app(app)
-    api_rest.init_app(app, title='EveAI API', version='1.0', description='EveAI API')
+    api_rest.init_app(app,
+                      title='EveAI API',
+                      version='1.0',
+                      description='EveAI API',
+                      doc='/api/v1/',
+                      prefix='/api/v1'),
     jwt.init_app(app)
     minio_client.init_app(app)
     simple_encryption.init_app(app)
+    cache_manager.init_app(app)
     cors.init_app(app, resources={
         r"/api/v1/*": {
             "origins": "*",
@@ -122,7 +131,7 @@ def register_extensions(app):
             "expose_headers": ["Content-Length", "Content-Range"],
             "supports_credentials": True,
             "max_age": 1728000,  # 20 days
-            "allow_credentials": True
+            # "allow_credentials": True
         }
     })
 
@@ -130,6 +139,7 @@ def register_extensions(app):
 def register_namespaces(app):
     api_rest.add_namespace(document_ns, path='/api/v1/documents')
     api_rest.add_namespace(auth_ns, path='/api/v1/auth')
+    api_rest.add_namespace(specialist_execution_ns, path='/api/v1/specialist-execution')
 
 
 def register_blueprints(app):
@@ -194,3 +204,8 @@ def register_error_handlers(app):
             "message": str(e),
             "type": "BadRequestError"
         }), 400
+
+
+def register_cache_handlers(app):
+    from common.utils.cache.config_cache import register_config_cache_handlers
+    register_config_cache_handlers(cache_manager)

eveai_api/api/auth.py

@@ -83,7 +83,6 @@ class Token(Resource):
                 expires_delta=expires_delta,
                 additional_claims=additional_claims
             )
-            current_app.logger.debug(f"Created token: {access_token}")
             return {
                 'access_token': access_token,
                 'expires_in': expires_delta.total_seconds()
@@ -164,10 +163,6 @@ class Services(Resource):
         """
         Get allowed services for the current token
         """
-        # Log the incoming authorization header
-        auth_header = request.headers.get('Authorization')
-        current_app.logger.debug(f"Received Authorization header: {auth_header}")
-
         claims = get_jwt()
         tenant_id = get_jwt_identity()
 

eveai_api/api/document_api.py

@@ -1,19 +1,23 @@
 import io
 import json
 from datetime import datetime
+from typing import Tuple, Any
 
 import pytz
 import requests
 from flask import current_app, request
 from flask_restx import Namespace, Resource, fields, reqparse
 from flask_jwt_extended import jwt_required, get_jwt_identity
+from sqlalchemy import desc
 from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
+
+from common.models.document import DocumentVersion
 from common.utils.document_utils import (
     create_document_stack, process_url, start_embedding_task,
     EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
     get_documents_list, edit_document, refresh_document, edit_document_version,
-    refresh_document_with_info, lookup_document, refresh_document_with_content
+    refresh_document_with_info, lookup_document, refresh_document_with_content, clean_url
 )
 from common.utils.eveai_exceptions import EveAIException
 from eveai_api.api.auth import requires_service
@@ -37,7 +41,8 @@ document_ns = Namespace('documents', description='Document related operations')
 
 # Define models for request parsing and response serialization
 upload_parser = reqparse.RequestParser()
-upload_parser.add_argument('catalog_id', location='form', type=int, required=True, help='The catalog to add the file to')
+upload_parser.add_argument('catalog_id', location='form', type=int, required=True,
+                           help='The catalog to add the file to')
 upload_parser.add_argument('file', location='files', type=FileStorage, required=True, help='The file to upload')
 upload_parser.add_argument('name', location='form', type=str, required=False, help='Name of the document')
 upload_parser.add_argument('language', location='form', type=str, required=True, help='Language of the document')
@@ -69,7 +74,11 @@ class AddDocument(Resource):
     @document_ns.response(500, 'Internal Server Error')
     def post(self):
         """
-        Add a new document by providing the content of a file (Multipart/form-data).
+        Upload a new document to EveAI by directly providing the file content.
+
+        This endpoint accepts multipart/form-data with the file content and metadata. It processes
+        the file, creates a new document in the specified catalog, and initiates the embedding
+        process.
         """
         tenant_id = get_jwt_identity()
         current_app.logger.info(f'Adding document for tenant {tenant_id}')
@@ -126,10 +135,11 @@ add_document_through_url = document_ns.model('AddDocumentThroughURL', {
     'valid_from': fields.String(required=False, description='Valid from date for the document'),
     'user_metadata': fields.String(required=False, description='User metadata for the document'),
     'system_metadata': fields.String(required=False, description='System metadata for the document'),
-    'catalog_properties': fields.String(required=False, description='The catalog configuration to be passed along (JSON '
-                                                                    'format). Validity is against catalog requirements '
-                                                                    'is not checked, and is the responsibility of the '
-                                                                    'calling client.'),
+    'catalog_properties': fields.String(required=False,
+                                        description='The catalog configuration to be passed along (JSON '
+                                                    'format). Validity is against catalog requirements '
+                                                    'is not checked, and is the responsibility of the '
+                                                    'calling client.'),
 })
 
 add_document_through_url_response = document_ns.model('AddDocumentThroughURLResponse', {
@@ -139,6 +149,7 @@ add_document_through_url_response = document_ns.model('AddDocumentThroughURLResp
     'task_id': fields.String(description='ID of the embedding task')
 })
 
+
 @document_ns.route('/add_document_through_url')
 class AddDocumentThroughURL(Resource):
     @jwt_required()
@@ -150,8 +161,10 @@ class AddDocumentThroughURL(Resource):
     @document_ns.response(500, 'Internal Server Error')
     def post(self):
         """
-        Add a new document using a URL. The URL can be temporary, and will not be stored.
-        Mainly used for passing temporary URLs like used in e.g. Zapier
+        Add a new document to EveAI using a temporary URL.
+
+        This endpoint is primarily used for integration with services that provide temporary URLs
+        (like Zapier). The URL content is downloaded and processed as a new document.
         """
         tenant_id = get_jwt_identity()
         current_app.logger.info(f'Adding document through url for tenant {tenant_id}')
@@ -164,29 +177,19 @@ class AddDocumentThroughURL(Resource):
             raise
 
         try:
-            # Step 1: Download from stashed URL
-            stashed_url = args['temp_url']
-            current_app.logger.info(f"Downloading stashed file from URL: {stashed_url}")
-            response = requests.get(stashed_url, stream=True)
-            response.raise_for_status()
-
-            hydration_url = response.text.strip()
-            current_app.logger.info(f"Downloading actual file from URL: {hydration_url}")
-            # Step 2: Download from hydration URL
-            actual_file_response = requests.get(hydration_url, stream=True)
-            actual_file_response.raise_for_status()
-            hydrated_file_content = actual_file_response.content
+            user_metadata = json.loads(args.get('user_metadata', '{}'))
+            actual_file_content, actual_file_content_type = download_file_content(args['temp_url'], user_metadata)
 
             # Get filename from URL or use provided name
             filename = secure_filename(args.get('name'))
             extension = filename.rsplit('.', 1)[1].lower() if '.' in filename else ''
 
             # Create FileStorage object from downloaded content
-            file_content = io.BytesIO(hydrated_file_content)
+            file_content = io.BytesIO(actual_file_content)
             file = FileStorage(
                 stream=file_content,
                 filename=filename,
-                content_type=response.headers.get('content-type', 'application/octet-stream')
+                content_type=actual_file_content_type
             )
 
             current_app.logger.info(f"Successfully downloaded file: {filename}")
@@ -233,10 +236,11 @@ add_url_model = document_ns.model('AddURL', {
     'valid_from': fields.String(required=False, description='Valid from date for the document'),
     'user_metadata': fields.String(required=False, description='User metadata for the document'),
     'system_metadata': fields.String(required=False, description='System metadata for the document'),
-    'catalog_properties': fields.String(required=False, description='The catalog configuration to be passed along (JSON '
-                                                                    'format). Validity is against catalog requirements '
-                                                                    'is not checked, and is the responsibility of the '
-                                                                    'calling client.'),
+    'catalog_properties': fields.String(required=False,
+                                        description='The catalog configuration to be passed along (JSON '
+                                                    'format). Validity is against catalog requirements '
+                                                    'is not checked, and is the responsibility of the '
+                                                    'calling client.'),
 })
 
 add_url_response = document_ns.model('AddURLResponse', {
@@ -257,19 +261,22 @@ class AddURL(Resource):
     @document_ns.response(500, 'Internal Server Error')
     def post(self):
         """
-        Add a new document from URL. The URL in this case is stored and can be used to refresh the document.
-        As a consequence, this must be a permanent and accessible URL.
+        Add a new document to EveAI from a permanent URL.
+
+        This endpoint is used for URLs that will remain accessible. The URL is stored and can
+        be used to refresh the document's content later.
         """
         tenant_id = get_jwt_identity()
         current_app.logger.info(f'Adding document from URL for tenant {tenant_id}')
 
         try:
             args = document_ns.payload
-            file_content, filename, extension = process_url(args['url'], tenant_id)
+            cleaned_url = clean_url(args['url'])
+            file_content, filename, extension = process_url(cleaned_url, tenant_id)
 
             api_input = {
                 'catalog_id': args['catalog_id'],
-                'url': args['url'],
+                'url': cleaned_url,
                 'name': args.get('name') or filename,
                 'language': args['language'],
                 'user_context': args.get('user_context'),
@@ -339,7 +346,6 @@ class DocumentResource(Resource):
     def put(self, document_id):
         """Edit a document. The content of the document will not be refreshed!"""
         try:
-            current_app.logger.debug(f'Editing document {document_id}')
             data = request.json
             tenant_id = get_jwt_identity()
             updated_doc, error = edit_document(tenant_id, document_id, data.get('name', None),
@@ -383,7 +389,8 @@ class DocumentVersionResource(Resource):
         """Edit a document version"""
         data = request.json
         tenant_id = get_jwt_identity()
-        updated_version, error = edit_document_version(tenant_id, version_id, data['user_context'], data.get('catalog_properties'))
+        updated_version, error = edit_document_version(tenant_id, version_id, data['user_context'],
+                                                       data.get('catalog_properties'))
         if updated_version:
             return {'message': f'Document Version {updated_version.id} updated successfully'}, 200
         else:
@@ -518,58 +525,87 @@ class DocumentLookup(Resource):
             return {'message': f'Missing required field: {str(e)}'}, 400
 
 
-refresh_content_model = document_ns.model('RefreshDocumentContent', {
-    'file_content': fields.Raw(required=True, description='The new file content'),
+refresh_url_model = document_ns.model('RefreshDocumentThroughURL', {
+    'temp_url': fields.String(required=True, description='Temporary URL of the updated document content'),
     'language': fields.String(required=False, description='Language of the document'),
     'user_context': fields.String(required=False, description='User context for the document'),
     'user_metadata': fields.Raw(required=False, description='Custom metadata fields'),
     'catalog_properties': fields.Raw(required=False, description='Catalog-specific properties'),
-    'trigger_service': fields.String(required=False, description='Service that triggered the update')
 })
 
 
-@document_ns.route('/<int:document_id>/refresh_content')
-class RefreshDocumentContent(Resource):
+@document_ns.route('/<int:document_id>/refresh_through_url')
+class RefreshDocumentThroughURL(Resource):
     @jwt_required()
     @requires_service('DOCAPI')
-    @document_ns.expect(refresh_content_model)
+    @document_ns.expect(refresh_url_model)
     @document_ns.response(200, 'Document refreshed successfully')
     def post(self, document_id):
-        """Refresh a document with new content"""
+        """Refresh a document using content from a URL"""
         tenant_id = get_jwt_identity()
+        current_app.logger.info(f'Refreshing document {document_id} through URL for tenant {tenant_id}')
+
         try:
-            data = request.json
-            file_content = data['file_content']
+            # Get filename from the existing version
+            old_doc_vers = (DocumentVersion.query.filter_by(doc_id=document_id).
+                            order_by(desc(DocumentVersion.id)).first())
+            filename = f"{old_doc_vers.id}.{old_doc_vers.file_type}"
 
-            # Build user_metadata by merging:
-            # 1. Existing metadata (if any)
-            # 2. New metadata from request
-            # 3. Zapier-specific fields
-            user_metadata = data.get('user_metadata', {})
-            user_metadata.update({
-                'source': 'zapier',
-                'trigger_service': data.get('trigger_service')
-            })
-            data['user_metadata'] = user_metadata
+            args = request.json
+            user_metadata = json.loads(args.get('user_metadata', '{}'))
 
-            # Keep catalog_properties separate
-            if 'catalog_properties' in data:
-                # We could add validation here against catalog configuration
-                data['catalog_properties'] = data['catalog_properties']
+            try:
+                actual_file_content, actual_file_content_type = download_file_content(args['temp_url'], user_metadata)
+                file_content = io.BytesIO(actual_file_content)
+                file = FileStorage(
+                    stream=file_content,
+                    filename=filename,
+                    content_type=actual_file_content_type
+                )
 
-            new_version, task_id = refresh_document_with_content(
-                document_id,
-                tenant_id,
-                file_content,
-                data
-            )
+                new_version, task_id = refresh_document_with_content(
+                    document_id,
+                    tenant_id,
+                    actual_file_content,
+                    args
+                )
 
-            return {
-                'message': f'Document refreshed successfully. New version: {new_version.id}. Task ID: {task_id}',
-                'document_id': document_id,
-                'document_version_id': new_version.id,
-                'task_id': task_id
-            }, 200
+                return {
+                    'message': f'Document refreshed successfully. New version: {new_version.id}. Task ID: {task_id}',
+                    'document_id': document_id,
+                    'document_version_id': new_version.id,
+                    'task_id': task_id
+                }, 200
+
+            except requests.RequestException as e:
+                current_app.logger.error(f"Error downloading file: {str(e)}")
+                return {'message': f'Error downloading file: {str(e)}'}, 422
 
         except EveAIException as e:
             return e.to_dict(), e.status_code
+
+
+def download_file_content(url: str, user_metadata: dict) -> tuple[Any, Any]:
+    if user_metadata and 'service' in user_metadata and 'Zapier' in user_metadata['service']:
+        # Zapier uses a system of Stashed URLs
+        # Step 1: Download from stashed URL
+        stashed_url = url
+        current_app.logger.info(f"Downloading stashed file from URL: {stashed_url}")
+        response = requests.get(stashed_url, stream=True)
+        response.raise_for_status()
+
+        hydration_url = response.text.strip()
+        current_app.logger.info(f"Downloading actual file from URL: {hydration_url}")
+        # Step 2: Download from hydration URL
+        actual_file_response = requests.get(hydration_url, stream=True)
+        actual_file_response.raise_for_status()
+        actual_file_content = actual_file_response.content
+    else:
+        actual_url = url
+        actual_file_response = requests.get(actual_url, stream=True)
+        actual_file_response.raise_for_status()
+        actual_file_content = actual_file_response.content
+
+    actual_file_content_type = actual_file_response.headers.get('content-type', 'application/octet-stream')
+
+    return actual_file_content, actual_file_content_type

eveai_api/api/specialist_execution_api.py

@@ -0,0 +1,134 @@
+# eveai_api/api/specialist_execution_api.py
+import uuid
+
+from flask import Response, stream_with_context, current_app
+from flask_restx import Namespace, Resource, fields
+from flask_jwt_extended import jwt_required, get_jwt_identity
+
+from common.extensions import cache_manager
+from common.utils.celery_utils import current_celery
+from common.utils.execution_progress import ExecutionProgressTracker
+from eveai_api.api.auth import requires_service
+from common.models.interaction import Specialist
+
+specialist_execution_ns = Namespace('specialist-execution', description='Specialist execution operations')
+
+specialist_start_session_response = specialist_execution_ns.model('StartSessionResponse', {
+    'session_id': fields.String(required=True, description='A new Chat session ID'),
+})
+
+
+@specialist_execution_ns.route('/start_session', methods=['GET'])
+class StartSession(Resource):
+    @jwt_required()
+    @requires_service("SPECIALIST_API")
+    @specialist_execution_ns.response(201, 'New Session ID created Successfully', specialist_start_session_response)
+    def get(self):
+        new_session_id = f"{uuid.uuid4()}"
+        return {
+            'session_id': new_session_id,
+        }, 201
+
+
+specialist_execution_input = specialist_execution_ns.model('SpecialistExecutionInput', {
+    'specialist_id': fields.Integer(required=True, description='ID of the specialist to use'),
+    'arguments': fields.Raw(required=True, description='Dynamic arguments for specialist and retrievers'),
+    'session_id': fields.String(required=True, description='Chat session ID'),
+    'user_timezone': fields.String(required=True, description='User timezone')
+})
+
+specialist_execution_response = specialist_execution_ns.model('SpecialistExecutionResponse', {
+    'task_id': fields.String(description='ID of specialist execution task, to be used to retrieve execution stream'),
+    'status': fields.String(description='Status of the execution'),
+    'stream_url': fields.String(description='Stream URL'),
+})
+
+
+@specialist_execution_ns.route('')
+class StartExecution(Resource):
+    @jwt_required()
+    @requires_service('SPECIALIST_API')
+    @specialist_execution_ns.expect(specialist_execution_input)
+    @specialist_execution_ns.response(201, 'Specialist execution successfully queued.', specialist_execution_response)
+    def post(self):
+        """Start execution of a specialist"""
+        tenant_id = get_jwt_identity()
+        data = specialist_execution_ns.payload
+
+        # Send task to queue
+        task = current_celery.send_task(
+            'execute_specialist',
+            args=[tenant_id,
+                  data['specialist_id'],
+                  data['arguments'],
+                  data['session_id'],
+                  data['user_timezone'],
+                  ],
+            queue='llm_interactions'
+        )
+
+        return {
+            'task_id': task.id,
+            'status': 'queued',
+            'stream_url': f'/api/v1/specialist-execution/{task.id}/stream'
+        }, 201
+
+
+@specialist_execution_ns.route('/<string:task_id>/stream')
+class ExecutionStream(Resource):
+    @jwt_required()
+    @requires_service('SPECIALIST_API')
+    def get(self, task_id: str):
+        """Get streaming updates for a specialist execution"""
+        progress_tracker = ExecutionProgressTracker()
+        return Response(
+            stream_with_context(progress_tracker.get_updates(task_id)),
+            mimetype='text/event-stream',
+            headers={
+                'Cache-Control': 'no-cache',
+                'Connection': 'keep-alive'
+            }
+        )
+
+
+specialist_arguments_input = specialist_execution_ns.model('SpecialistArgumentsInput', {
+    'specialist_id': fields.Integer(required=True, description='ID of the specialist to use'),
+})
+
+specialist_arguments_response = specialist_execution_ns.model('SpecialistArgumentsResponse', {
+    'arguments': fields.Raw(description='Dynamic list of attributes for the specialist.'),
+})
+
+
+@specialist_execution_ns.route('/specialist_arguments', methods=['GET'])
+class SpecialistArgument(Resource):
+    @jwt_required()
+    @requires_service('SPECIALIST_API')
+    @specialist_execution_ns.expect(specialist_arguments_input)
+    @specialist_execution_ns.response(200, 'Specialist configuration fetched.', specialist_arguments_response)
+    @specialist_execution_ns.response(404, 'Specialist configuration not found.')
+    @specialist_execution_ns.response(500, 'Internal Server Error')
+    def get(self):
+        """Start execution of a specialist"""
+        tenant_id = get_jwt_identity()
+        data = specialist_execution_ns.payload
+        specialist_id = data['specialist_id']
+        try:
+            specialist = Specialist.query.get(specialist_id)
+            if specialist:
+                configuration = cache_manager.specialists_config_cache.get_config(specialist.type,
+                                                                                  specialist.type_version)
+                if configuration:
+                    if 'arguments' in configuration:
+                        return {
+                            'arguments': configuration['arguments'],
+                        }, 200
+                    else:
+                        specialist_execution_ns.abort(404, 'No arguments found in specialist configuration.')
+                else:
+                    specialist_execution_ns.abort(404, 'Error fetching Specialist configuration.')
+            else:
+                specialist_execution_ns.abort(404, 'Error fetching Specialist')
+        except Exception as e:
+            current_app.logger.error(f"Error while retrieving Specialist configuration: {str(e)}")
+            specialist_execution_ns.abort(500, 'Unexpected Error while fetching Specialist configuration.')

eveai_app/__init__.py

@@ -1,7 +1,7 @@
 import logging
 import os
-from flask import Flask, render_template, jsonify, flash, redirect, request
-from flask_security import SQLAlchemyUserDatastore, LoginForm
+from flask import Flask, jsonify
+from flask_security import SQLAlchemyUserDatastore
 from flask_security.signals import user_authenticated
 from werkzeug.middleware.proxy_fix import ProxyFix
 import logging.config
@@ -12,7 +12,7 @@ from common.models.user import User, Role, Tenant, TenantDomain
 import common.models.interaction
 import common.models.entitlements
 import common.models.document
-from common.utils.nginx_utils import prefixed_url_for
+from common.utils.startup_eveai import perform_startup_actions
 from config.logging_config import LOGGING
 from common.utils.security import set_tenant_session_data
 from .errors import register_error_handlers
@@ -73,6 +73,9 @@ def create_app(config_file=None):
     # Register Error Handlers
     register_error_handlers(app)
 
+    # Register Cache Handlers
+    register_cache_handlers(app)
+
     # Debugging settings
     if app.config['DEBUG'] is True:
         app.logger.setLevel(logging.DEBUG)
@@ -103,7 +106,19 @@ def create_app(config_file=None):
     # Register template filters
     register_filters(app)
 
-    app.logger.info("EveAI App Server Started Successfully")
+    # Let all initialization complete before using cache
+    # @app.before_first_request
+    # def initialize_cache_data():
+    #     # Cache testing
+    #     agent_types = cache_manager.agent_config_cache.get_types()
+    #     app.logger.debug(f"Agent types: {agent_types}")
+    #     agent_config = cache_manager.agent_config_cache.get_config('RAG_AGENT')
+    #     app.logger.debug(f"Agent config: {agent_config}")
+
+    # Perform startup actions such as cache invalidation
+    perform_startup_actions(app)
+
+    app.logger.info(f"EveAI App Server Started Successfully (PID: {os.getpid()})")
     app.logger.info("-------------------------------------------------------------------------------------------------")
     return app
 
@@ -123,7 +138,6 @@ def register_extensions(app):
     metrics.init_app(app)
 
 
-# Register Blueprints
 def register_blueprints(app):
     from .views.user_views import user_bp
     app.register_blueprint(user_bp)
@@ -143,3 +157,14 @@ def register_blueprints(app):
     app.register_blueprint(healthz_bp)
     init_healtz(app)
 
+
+def register_cache_handlers(app):
+    from common.utils.cache.config_cache import register_config_cache_handlers
+    register_config_cache_handlers(cache_manager)
+    from common.utils.cache.crewai_processed_config_cache import register_specialist_cache_handlers
+    register_specialist_cache_handlers(cache_manager)
+
+
+
+
+

eveai_app/errors.py

@@ -1,3 +1,4 @@
+import jinja2
 from flask import render_template, request, jsonify, redirect, current_app
 from flask_login import current_user
 from common.utils.nginx_utils import prefixed_url_for
@@ -47,3 +48,15 @@ def register_error_handlers(app):
     app.register_error_handler(403, not_authorised_error)
     app.register_error_handler(KeyError, key_error_handler)
 
+    @app.errorhandler(jinja2.TemplateNotFound)
+    def template_not_found(error):
+        app.logger.error(f'Template not found: {error.name}')
+        app.logger.error(f'Search Paths: {app.jinja_loader.list_templates()}')
+        return f'Template not found: {error.name}. Check logs for details.', 404
+
+    @app.errorhandler(jinja2.TemplateSyntaxError)
+    def template_syntax_error(error):
+        app.logger.error(f'Template syntax error: {error.message}')
+        app.logger.error(f'In template {error.filename}, line {error.lineno}')
+        return f'Template syntax error: {error.message}', 500
+

eveai_app/templates/document/edit_catalog.html

@@ -11,7 +11,7 @@ When you change chunking of embedding information, you'll need to manually refre
 {% block content %}
     <form method="post">
         {{ form.hidden_tag() }}
-        {% set disabled_fields = ['type'] %}
+        {% set disabled_fields = ['type', 'embedding_model'] %}
         {% set exclude_fields = [] %}
         <!-- Render Static Fields -->
         {% for field in form.get_static_fields() %}