Changelog update for 2.3.8-alfa

- Add possibility to add allowed_languages to tenant make

README.md.k8s-logging

@@ -0,0 +1,67 @@
+# Kubernetes Logging Upgrade
+
+## Overzicht
+Deze instructies beschrijven hoe je alle services moet bijwerken om de nieuwe logging configuratie te gebruiken die zowel compatibel is met traditionele bestandsgebaseerde logging (voor ontwikkeling/test) als met Kubernetes (voor productie).
+
+## Stappen voor elke service
+
+Pas de volgende wijzigingen toe in elk van de volgende services:
+
+- eveai_app
+- eveai_workers
+- eveai_api
+- eveai_chat_client
+- eveai_chat_workers
+- eveai_beat
+- eveai_entitlements
+
+### 1. Update de imports
+
+Verander:
+```python
+from config.logging_config import LOGGING
+```
+
+Naar:
+```python
+from config.logging_config import configure_logging
+```
+
+### 2. Update de logging configuratie
+
+Verander:
+```python
+logging.config.dictConfig(LOGGING)
+```
+
+Naar:
+```python
+configure_logging()
+```
+
+## Dockerfile Aanpassingen
+
+Voeg de volgende regels toe aan je Dockerfile voor elke service om de Kubernetes-specifieke logging afhankelijkheden te installeren (alleen voor productie):
+
+```dockerfile
+# Alleen voor productie (Kubernetes) builds
+COPY requirements-k8s.txt /app/
+RUN if [ "$ENVIRONMENT" = "production" ]; then pip install -r requirements-k8s.txt; fi
+```
+
+## Kubernetes Deployment
+
+Zorg ervoor dat je Kubernetes deployment manifests de volgende omgevingsvariabele bevatten:
+
+```yaml
+env:
+  - name: FLASK_ENV
+    value: "production"
+```
+
+## Voordelen
+
+1. De code detecteert automatisch of deze in Kubernetes draait
+2. In ontwikkeling/test omgevingen blijft alles naar bestanden schrijven
+3. In Kubernetes gaan logs naar stdout/stderr in JSON-formaat
+4. Geen wijzigingen nodig in bestaande logger code in de applicatie

common/langchain/llm_metrics_handler.py

@@ -3,7 +3,6 @@ from langchain.callbacks.base import BaseCallbackHandler
 from typing import Dict, Any, List
 from langchain.schema import LLMResult
 from common.utils.business_event_context import current_event
-from flask import current_app
 
 
 class LLMMetricsHandler(BaseCallbackHandler):

common/langchain/persistent_llm_metrics_handler.py

@@ -0,0 +1,47 @@
+import time
+from langchain.callbacks.base import BaseCallbackHandler
+from typing import Dict, Any, List
+from langchain.schema import LLMResult
+from common.utils.business_event_context import current_event
+
+
+class PersistentLLMMetricsHandler(BaseCallbackHandler):
+    """Metrics handler that allows metrics to be retrieved from within any call. In case metrics are required for other
+    purposes than business event logging."""
+
+    def __init__(self):
+        self.total_tokens: int = 0
+        self.prompt_tokens: int = 0
+        self.completion_tokens: int = 0
+        self.start_time: float = 0
+        self.end_time: float = 0
+        self.total_time: float = 0
+
+    def reset(self):
+        self.total_tokens = 0
+        self.prompt_tokens = 0
+        self.completion_tokens = 0
+        self.start_time = 0
+        self.end_time = 0
+        self.total_time = 0
+
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        self.start_time = time.time()
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        self.end_time = time.time()
+        self.total_time = self.end_time - self.start_time
+
+        usage = response.llm_output.get('token_usage', {})
+        self.prompt_tokens += usage.get('prompt_tokens', 0)
+        self.completion_tokens += usage.get('completion_tokens', 0)
+        self.total_tokens = self.prompt_tokens + self.completion_tokens
+
+    def get_metrics(self) -> Dict[str, int | float]:
+        return {
+            'total_tokens': self.total_tokens,
+            'prompt_tokens': self.prompt_tokens,
+            'completion_tokens': self.completion_tokens,
+            'time_elapsed': self.total_time,
+            'interaction_type': 'LLM',
+        }

common/models/document.py

@@ -34,6 +34,7 @@ class Processor(db.Model):
     catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True)
     type = db.Column(db.String(50), nullable=False)
     sub_file_type = db.Column(db.String(50), nullable=True)
+    active = db.Column(db.Boolean, nullable=True, default=True)
 
     # Tuning enablers
     tuning = db.Column(db.Boolean, nullable=True, default=False)

common/models/user.py

@@ -186,6 +186,7 @@ class TenantMake(db.Model):
     active = db.Column(db.Boolean, nullable=False, default=True)
     website = db.Column(db.String(255), nullable=True)
     logo_url = db.Column(db.String(255), nullable=True)
+    allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True)
 
     # Chat customisation options
     chat_customisation_options = db.Column(JSONB, nullable=True)
@@ -317,3 +318,27 @@ class SpecialistMagicLinkTenant(db.Model):
 
     magic_link_code = db.Column(db.String(55), primary_key=True)
     tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+
+
+class TranslationCache(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    cache_key = db.Column(db.String(16), primary_key=True)
+    source_text = db.Column(db.Text, nullable=False)
+    translated_text = db.Column(db.Text, nullable=False)
+    source_language = db.Column(db.String(2), nullable=False)
+    target_language = db.Column(db.String(2), nullable=False)
+    context = db.Column(db.Text, nullable=True)
+
+    # Translation cost
+    prompt_tokens = db.Column(db.Integer, nullable=False)
+    completion_tokens = db.Column(db.Integer, nullable=False)
+
+    # Tracking
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
+
+    last_used_at = db.Column(db.DateTime, nullable=True)

common/services/utils/translation_services.py

@@ -0,0 +1,43 @@
+import xxhash
+import json
+
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+
+from common.langchain.persistent_llm_metrics_handler import PersistentLLMMetricsHandler
+from common.utils.model_utils import get_template, replace_variable_in_template
+
+class TranslationService:
+    def __init__(self, tenant_id):
+        self.tenant_id = tenant_id
+
+    def translate_text(self, text_to_translate: str, target_lang: str, source_lang: str = None, context: str = None) -> tuple[
+        str, dict[str, int | float]]:
+        prompt_params = {
+            "text_to_translate": text_to_translate,
+            "target_lang": target_lang,
+        }
+        if context:
+            template, llm = get_template("translation_with_context")
+            prompt_params["context"] = context
+        else:
+            template, llm = get_template("translation_without_context")
+
+        # Add a metrics handler to capture usage
+
+        metrics_handler = PersistentLLMMetricsHandler()
+        existing_callbacks = llm.callbacks
+        llm.callbacks = existing_callbacks + [metrics_handler]
+
+        translation_prompt = ChatPromptTemplate.from_template(template)
+
+        setup = RunnablePassthrough()
+
+        chain = (setup | translation_prompt | llm | StrOutputParser())
+
+        translation = chain.invoke(prompt_params)
+
+        metrics = metrics_handler.get_metrics()
+
+        return translation, metrics

common/utils/cache/translation_cache.py

@@ -0,0 +1,156 @@
+import json
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+
+import xxhash
+from flask import current_app
+from sqlalchemy import and_
+from sqlalchemy.inspection import inspect
+
+from common.utils.cache.base import CacheHandler, T
+from common.extensions import db
+
+from common.models.user import TranslationCache
+from common.services.utils.translation_services import TranslationService
+from flask_security import current_user
+
+
+class TranslationCacheHandler(CacheHandler[TranslationCache]):
+    """Handles caching of translations with fallback to database and external translation service"""
+    handler_name = 'translation_cache'
+
+    def __init__(self, region):
+        super().__init__(region, 'translation')
+        self.configure_keys('hash_key')
+
+    def _to_cache_data(self, instance: TranslationCache) -> Dict[str, Any]:
+        """Convert TranslationCache instance to cache data using SQLAlchemy inspection"""
+        if not instance:
+            return {}
+
+        mapper = inspect(TranslationCache)
+        data = {}
+
+        for column in mapper.columns:
+            value = getattr(instance, column.name)
+
+            # Handle date serialization
+            if isinstance(value, dt):
+                data[column.name] = value.isoformat()
+            else:
+                data[column.name] = value
+
+        return data
+
+    def _from_cache_data(self, data: Dict[str, Any], **kwargs) -> TranslationCache:
+        if not data:
+            return None
+
+        # Create a new TranslationCache instance
+        translation = TranslationCache()
+        mapper = inspect(TranslationCache)
+
+        # Set all attributes dynamically
+        for column in mapper.columns:
+            if column.name in data:
+                value = data[column.name]
+
+                # Handle date deserialization
+                if column.name.endswith('_date') and value:
+                    if isinstance(value, str):
+                        value = dt.fromisoformat(value).date()
+
+                setattr(translation, column.name, value)
+
+        return translation
+
+    def _should_cache(self, value: TranslationCache) -> bool:
+        """Validate if the translation should be cached"""
+        return value is not None and value.cache_key is not None
+
+    def get_translation(self, text: str, target_lang: str, source_lang:str=None, context: str=None) -> Optional[TranslationCache]:
+        """
+        Get the translation for a text in a specific language
+
+        Args:
+            text: The text to be translated
+            target_lang: The target language for the translation
+            source_lang: The source language of the text to be translated
+            context: Optional context for the translation
+
+        Returns:
+            TranslationCache instance if found, None otherwise
+        """
+
+        def creator_func(text: str, target_lang: str, source_lang: str=None, context: str=None) -> Optional[TranslationCache]:
+            # Generate cache key based on inputs
+            cache_key = self._generate_cache_key(text, target_lang, source_lang, context)
+
+            # Check if translation already exists in database
+            existing_translation = db.session.query(TranslationCache).filter_by(cache_key=cache_key).first()
+
+            if existing_translation:
+                # Update last used timestamp
+                existing_translation.last_used_at = dt.now(tz=tz.utc)
+                db.session.commit()
+                return existing_translation
+
+            # Translation not found in DB, need to create it
+            # Initialize translation service
+            translation_service = TranslationService(getattr(current_app, 'tenant_id', None))
+
+            # Get the translation and metrics
+            translated_text, metrics = translation_service.translate_text(
+                text_to_translate=text,
+                target_lang=target_lang,
+                source_lang=source_lang,
+                context=context
+            )
+
+            # Create new translation cache record
+            new_translation = TranslationCache(
+                cache_key=cache_key,
+                source_text=text,
+                translated_text=translated_text,
+                source_language=source_lang or 'auto',
+                target_language=target_lang,
+                context=context,
+                prompt_tokens=metrics.get('prompt_tokens', 0),
+                completion_tokens=metrics.get('completion_tokens', 0),
+                created_at=dt.now(tz=tz.utc),
+                created_by=getattr(current_user, 'id', None) if 'current_user' in globals() else None,
+                updated_at=dt.now(tz=tz.utc),
+                updated_by=getattr(current_user, 'id', None) if 'current_user' in globals() else None,
+                last_used_at=dt.now(tz=tz.utc)
+            )
+
+            # Save to database
+            db.session.add(new_translation)
+            db.session.commit()
+
+            return new_translation
+
+        return self.get(creator_func, text=text, target_lang=target_lang, source_lang=source_lang, context=context)
+
+    def invalidate_tenant_translations(self, tenant_id: int):
+        """Invalidate cached translations for specific tenant"""
+        self.invalidate(tenant_id=tenant_id)
+
+    def _generate_cache_key(self, text: str, target_lang: str, source_lang: str = None, context: str = None) -> str:
+        """Generate cache key for a translation"""
+        cache_data = {
+            "text": text.strip(),
+            "target_lang": target_lang.lower(),
+            "source_lang": source_lang.lower() if source_lang else None,
+            "context": context.strip() if context else None,
+        }
+
+        cache_string = json.dumps(cache_data, sort_keys=True, ensure_ascii=False)
+        return xxhash.xxh64(cache_string.encode('utf-8')).hexdigest()
+
+def register_translation_cache_handlers(cache_manager) -> None:
+    """Register translation cache handlers with cache manager"""
+    cache_manager.register_handler(
+        TranslationCacheHandler,
+        'eveai_model'  # Use existing eveai_model region
+    )

common/utils/chat_utils.py

@@ -1,14 +1,18 @@
+import json
+
 """
 Utility functions for chat customization.
 """
+from flask import current_app
+
 
 def get_default_chat_customisation(tenant_customisation=None):
     """
     Get chat customization options with default values for missing options.
 
     Args:
-        tenant_customization (dict, optional): The tenant's customization options.
+        tenant_customisation (dict or str, optional): The tenant's customization options.
-            Defaults to None.
+            Defaults to None. Can be a dict or a JSON string.
 
     Returns:
         dict: A dictionary containing all customization options with default values
@@ -37,7 +41,18 @@ def get_default_chat_customisation(tenant_customisation=None):
     # Start with the default customization
     customisation = default_customisation.copy()
 
+    # Convert JSON string to dict if needed
+    if isinstance(tenant_customisation, str):
+        try:
+            tenant_customisation = json.loads(tenant_customisation)
+            current_app.logger.debug(f"Converted JSON string to dict: {tenant_customisation}")
+        except json.JSONDecodeError as e:
+            current_app.logger.error(f"Error parsing JSON customisation: {e}")
+            return default_customisation
+
     # Update with tenant customization
+    current_app.logger.debug(f"Tenant customisation - in default creation: {tenant_customisation}")
+    if tenant_customisation:
         for key, value in tenant_customisation.items():
             if key in customisation:
                 customisation[key] = value

common/utils/document_utils.py

@@ -3,7 +3,7 @@ from datetime import datetime as dt, timezone as tz
 from sqlalchemy import desc
 from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.utils import secure_filename
-from common.models.document import Document, DocumentVersion, Catalog
+from common.models.document import Document, DocumentVersion, Catalog, Processor
 from common.extensions import db, minio_client
 from common.utils.celery_utils import current_celery
 from flask import current_app
@@ -11,6 +11,7 @@ import requests
 from urllib.parse import urlparse, unquote, urlunparse, parse_qs
 import os
 
+from config.type_defs.processor_types import PROCESSOR_TYPES
 from .config_field_types import normalize_json_field
 from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType,
                                EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException)
@@ -469,3 +470,15 @@ def lookup_document(tenant_id: int, lookup_criteria: dict, metadata_type: str) -
             "Error during document lookup",
             status_code=500
         )
+
+def is_file_type_supported_by_catalog(catalog_id, file_type):
+    processors = Processor.query.filter_by(catalog_id=catalog_id).filter_by(active=True).all()
+
+    supported_file_types = []
+    for processor in processors:
+        processor_file_types = PROCESSOR_TYPES[processor.type]['file_types']
+        file_types = [f.strip() for f in processor_file_types.split(",")]
+        supported_file_types.extend(file_types)
+
+    if file_type not in supported_file_types:
+        raise EveAIUnsupportedFileType()

common/utils/eveai_exceptions.py

@@ -34,7 +34,25 @@ class EveAIDoubleURLException(EveAIException):
 class EveAIUnsupportedFileType(EveAIException):
     """Raised when an invalid file type is provided"""
 
-    def __init__(self, message="Filetype is not supported", status_code=400, payload=None):
+    def __init__(self, message="Filetype is not supported by current active processors", status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoProcessorFound(EveAIException):
+    """Raised when no processor is found for a given file type"""
+
+    def __init__(self, catalog_id, file_type, file_subtype, status_code=400, payload=None):
+        message = f"No active processor found for catalog {catalog_id} with file type {file_type} and subtype {file_subtype}"
+        super().__init__(message, status_code, payload)
+
+
+class EveAINoContentFound(EveAIException):
+    """Raised when no content is found for a given document"""
+
+    def __init__(self, document_id, document_version_id, status_code=400, payload=None):
+        self.document_id = document_id
+        self.document_version_id = document_version_id
+        message = f"No content found while processing Document with ID {document_id} and version {document_version_id}."
         super().__init__(message, status_code, payload)
 
 
@@ -248,3 +266,14 @@ class EveAIPendingLicensePeriod(EveAIException):
         message = f"Basic Fee Payment has not been received yet. Please ensure payment has been made, and please wait for payment to be processed."
         super().__init__(message, status_code, payload)
 
+
+class EveAISpecialistExecutionError(EveAIException):
+    """Raised when an error occurs during specialist execution"""
+
+    def __init__(self, tenant_id, specialist_id, session_id, details, status_code=400, payload=None):
+        message = (f"Error during specialist {specialist_id} execution \n"
+                   f"with Session ID {session_id} \n"
+                   f"for Tenant {tenant_id}. \n"
+                   f"Details: {details} \n"
+                   f"The System Administrator has been notified. Please try again later.")
+        super().__init__(message, status_code, payload)

config/agents/traicie/TRAICIE_RECRUITER_AGENT/1.0.0.yaml

@@ -1,5 +1,5 @@
 version: "1.0.0"
-name: "Traicie HR BP  "
+name: "Traicie Recruiter"
 role: >
   You are an Expert Recruiter working for {tenant_name}
   {custom_role}
@@ -16,10 +16,10 @@ backstory: >
   AI-driven sourcing. You’re more than a recruiter—you’re a trusted advisor, a brand ambassador, and a connector of 
   people and purpose.
   {custom_backstory}
-full_model_name: "mistral.mistral-medium-latest"
+full_model_name: "mistral.magistral-medium-latest"
 temperature: 0.3
 metadata:
   author: "Josako"
-  date_added: "2025-05-21"
+  date_added: "2025-06-18"
-  description: "HR BP Agent."
+  description: "Traicie Recruiter Agent"
   changes: "Initial version"

config/config.py

@@ -12,10 +12,7 @@ class Config(object):
     DEBUG = False
     DEVELOPMENT = False
     SECRET_KEY = environ.get('SECRET_KEY')
-    SESSION_COOKIE_SECURE = False
-    SESSION_COOKIE_HTTPONLY = True
     COMPONENT_NAME = environ.get('COMPONENT_NAME')
-    SESSION_KEY_PREFIX = f'{COMPONENT_NAME}_'
 
     # Database Settings
     DB_HOST = environ.get('DB_HOST')
@@ -44,8 +41,6 @@ class Config(object):
     # SECURITY_POST_CHANGE_VIEW = '/admin/login'
     # SECURITY_BLUEPRINT_NAME = 'security_bp'
     SECURITY_PASSWORD_SALT = environ.get('SECURITY_PASSWORD_SALT')
-    REMEMBER_COOKIE_SAMESITE = 'strict'
-    SESSION_COOKIE_SAMESITE = 'Lax'
     SECURITY_CONFIRMABLE = True
     SECURITY_TRACKABLE = True
     SECURITY_PASSWORD_COMPLEXITY_CHECKER = 'zxcvbn'
@@ -56,6 +51,10 @@ class Config(object):
     SECURITY_EMAIL_SUBJECT_PASSWORD_NOTICE = 'Your Password Has Been Reset'
     SECURITY_EMAIL_PLAINTEXT = False
     SECURITY_EMAIL_HTML = True
+    SECURITY_SESSION_PROTECTION = 'basic'  # of 'basic' als 'strong' problemen geeft
+    SECURITY_REMEMBER_TOKEN_VALIDITY = timedelta(minutes=60)  # Zelfde als session lifetime
+    SECURITY_AUTO_LOGIN_AFTER_CONFIRM = True
+    SECURITY_AUTO_LOGIN_AFTER_RESET = True
 
     # Ensure Flask-Security-Too is handling CSRF tokens when behind a proxy
     SECURITY_CSRF_PROTECT_MECHANISMS = ['session']
@@ -149,7 +148,7 @@ class Config(object):
         },
     }
 
-    SUPPORTED_LANGUAGES_Full = list(SUPPORTED_LANGUAGE_DETAILS.keys())
+    SUPPORTED_LANGUAGES_FULL = list(SUPPORTED_LANGUAGE_DETAILS.keys())
 
     # supported currencies
     SUPPORTED_CURRENCIES = ['€', '$']
@@ -157,10 +156,7 @@ class Config(object):
     # supported LLMs
     # SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
     SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed']
-    SUPPORTED_LLMS = ['openai.gpt-4o', 'openai.gpt-4o-mini',
+    SUPPORTED_LLMS = ['mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest']
-                      'mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest']
-
-    ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', }
 
     # Annotation text chunk length
     ANNOTATION_TEXT_CHUNK_LENGTH = 10000
@@ -189,6 +185,15 @@ class Config(object):
     PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
     SESSION_REFRESH_EACH_REQUEST = True
 
+    SESSION_COOKIE_NAME = f'{COMPONENT_NAME}_session'
+    SESSION_COOKIE_DOMAIN = None  # Laat Flask dit automatisch bepalen
+    SESSION_COOKIE_PATH = '/'
+    SESSION_COOKIE_HTTPONLY = True
+    SESSION_COOKIE_SECURE = False  # True voor production met HTTPS
+    SESSION_COOKIE_SAMESITE = 'Lax'
+    REMEMBER_COOKIE_SAMESITE = 'strict'
+    SESSION_KEY_PREFIX = f'{COMPONENT_NAME}_'
+
     # JWT settings
     JWT_SECRET_KEY = environ.get('JWT_SECRET_KEY')
     JWT_ACCESS_TOKEN_EXPIRES = timedelta(hours=1)  # Set token expiry to 1 hour
@@ -267,6 +272,7 @@ class DevConfig(Config):
     # Define the nginx prefix used for the specific apps
     EVEAI_APP_LOCATION_PREFIX = '/admin'
     EVEAI_CHAT_LOCATION_PREFIX = '/chat'
+    CHAT_CLIENT_PREFIX = 'chat-client/chat/'
 
     # file upload settings
     # UPLOAD_FOLDER = '/app/tenant_files'

config/customisations/globals/CHAT_CLIENT_CUSTOMISATION/1.0.0.yaml

@@ -56,11 +56,6 @@ configuration:
     description: "Sidebar Markdown-formatted Text"
     type: "text"
     required: false
-  "welcome_message":
-    name: "Welcome Message"
-    description: "Text to be shown as Welcome"
-    type: "text"
-    required: false
 metadata:
   author: "Josako"
   date_added: "2024-06-06"

config/logging_config.py

@@ -1,15 +1,13 @@
+
 import json
 import os
+import sys
 from datetime import datetime as dt, timezone as tz
 
 from flask import current_app
-from graypy import GELFUDPHandler
 import logging
 import logging.config
 
-# Graylog configuration
-GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost')
-GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201))
 env = os.environ.get('FLASK_ENV', 'development')
 
 
@@ -144,23 +142,6 @@ class TuningFormatter(logging.Formatter):
         return formatted_msg
 
 
-class GraylogFormatter(logging.Formatter):
-    """Maintains existing Graylog formatting while adding tuning fields"""
-
-    def format(self, record):
-        if getattr(record, 'is_tuning_log', False):
-            # Add tuning-specific fields to Graylog
-            record.tuning_fields = {
-                'is_tuning_log': True,
-                'tuning_type': record.tuning_type,
-                'tenant_id': record.tenant_id,
-                'catalog_id': record.catalog_id,
-                'specialist_id': record.specialist_id,
-                'retriever_id': record.retriever_id,
-                'processor_id': record.processor_id,
-                'session_id': record.session_id,
-            }
-        return super().format(record)
 
 class TuningLogger:
     """Helper class to manage tuning logs with consistent structure"""
@@ -177,10 +158,10 @@ class TuningLogger:
             specialist_id: Optional specialist ID for context
             retriever_id: Optional retriever ID for context
             processor_id: Optional processor ID for context
-            session_id: Optional session ID for context and log file naming
+            session_id: Optional session ID for context
-            log_file: Optional custom log file name to use
+            log_file: Optional custom log file name (ignored - all logs go to tuning.log)
         """
-
+        # Always use the standard tuning logger
         self.logger = logging.getLogger(logger_name)
         self.tenant_id = tenant_id
         self.catalog_id = catalog_id
@@ -188,63 +169,8 @@ class TuningLogger:
         self.retriever_id = retriever_id
         self.processor_id = processor_id
         self.session_id = session_id
-        self.log_file = log_file
-        # Determine whether to use a session-specific logger
-        if session_id:
-            # Create a unique logger name for this session
-            session_logger_name = f"{logger_name}_{session_id}"
-            self.logger = logging.getLogger(session_logger_name)
 
-            # If this logger doesn't have handlers yet, configure it
+    def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DEBUG):
-            if not self.logger.handlers:
-                # Determine log file path
-                if not log_file and session_id:
-                    log_file = f"logs/tuning_{session_id}.log"
-                elif not log_file:
-                    log_file = "logs/tuning.log"
-
-                # Configure the logger
-                self._configure_session_logger(log_file)
-        else:
-            # Use the standard tuning logger
-            self.logger = logging.getLogger(logger_name)
-
-    def _configure_session_logger(self, log_file):
-        """Configure a new session-specific logger with appropriate handlers"""
-        # Create and configure a file handler
-        file_handler = logging.handlers.RotatingFileHandler(
-            filename=log_file,
-            maxBytes=1024 * 1024 * 3,  # 3MB
-            backupCount=3
-        )
-        file_handler.setFormatter(TuningFormatter())
-        file_handler.setLevel(logging.DEBUG)
-
-        # Add the file handler to the logger
-        self.logger.addHandler(file_handler)
-
-        # Add Graylog handler in production
-        env = os.environ.get('FLASK_ENV', 'development')
-        if env == 'production':
-            try:
-                graylog_handler = GELFUDPHandler(
-                    host=GRAYLOG_HOST,
-                    port=GRAYLOG_PORT,
-                    debugging_fields=True
-                )
-                graylog_handler.setFormatter(GraylogFormatter())
-                self.logger.addHandler(graylog_handler)
-            except Exception as e:
-                # Fall back to just file logging if Graylog setup fails
-                fallback_logger = logging.getLogger('eveai_app')
-                fallback_logger.warning(f"Failed to set up Graylog handler: {str(e)}")
-
-        # Set logger level and disable propagation
-        self.logger.setLevel(logging.DEBUG)
-        self.logger.propagate = False
-
-
-def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DEBUG):
         """Log a tuning event with structured data"""
         try:
             # Create a standard LogRecord for tuning
@@ -275,13 +201,82 @@ def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DE
             self.logger.handle(record)
 
         except Exception as e:
-            fallback_logger = logging.getLogger('eveai_workers')
+            print(f"Failed to log tuning message: {str(e)}")
-            fallback_logger.exception(f"Failed to log tuning message: {str(e)}")
 
 
 # Set the custom log record factory
 logging.setLogRecordFactory(TuningLogRecord)
 
+def configure_logging():
+    """Configure logging based on environment
+
+    When running in Kubernetes, directs logs to stdout in JSON format
+    Otherwise uses file-based logging for development/testing
+    """
+    try:
+        # Verkrijg het absolute pad naar de logs directory
+        base_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+        logs_dir = os.path.join(base_dir, 'logs')
+
+        # Zorg ervoor dat de logs directory bestaat met de juiste permissies
+        if not os.path.exists(logs_dir):
+            try:
+                os.makedirs(logs_dir, exist_ok=True)
+                print(f"Logs directory aangemaakt op: {logs_dir}")
+            except (IOError, PermissionError) as e:
+                print(f"WAARSCHUWING: Kan logs directory niet aanmaken: {e}")
+                print(f"Logs worden mogelijk niet correct geschreven!")
+
+        # Check if running in Kubernetes
+        in_kubernetes = os.environ.get('KUBERNETES_SERVICE_HOST') is not None
+
+        # Controleer of de pythonjsonlogger pakket beschikbaar is als we in Kubernetes zijn
+        if in_kubernetes:
+            try:
+                import pythonjsonlogger.jsonlogger
+                has_json_logger = True
+            except ImportError:
+                print("WAARSCHUWING: python-json-logger pakket is niet geïnstalleerd.")
+                print("Voer 'pip install python-json-logger>=2.0.7' uit om JSON logging in te schakelen.")
+                print("Terugvallen op standaard logging formaat.")
+                has_json_logger = False
+                in_kubernetes = False  # Fall back to standard logging
+        else:
+            has_json_logger = False
+
+        # Apply the configuration
+        logging_config = dict(LOGGING)
+
+        # Wijzig de json_console handler om terug te vallen op console als pythonjsonlogger niet beschikbaar is
+        if not has_json_logger and 'json_console' in logging_config['handlers']:
+            # Vervang json_console handler door een console handler met standaard formatter
+            logging_config['handlers']['json_console']['formatter'] = 'standard'
+
+        # In Kubernetes, conditionally modify specific loggers to use JSON console output
+        # This preserves the same logger names but changes where/how they log
+        if in_kubernetes:
+            for logger_name in logging_config['loggers']:
+                if logger_name:  # Skip the root logger
+                    logging_config['loggers'][logger_name]['handlers'] = ['json_console']
+
+        # Controleer of de logs directory schrijfbaar is voordat we de configuratie toepassen
+        logs_dir = os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs')
+        if os.path.exists(logs_dir) and not os.access(logs_dir, os.W_OK):
+            print(f"WAARSCHUWING: Logs directory bestaat maar is niet schrijfbaar: {logs_dir}")
+            print("Logs worden mogelijk niet correct geschreven!")
+
+        logging.config.dictConfig(logging_config)
+        logging.info(f"Logging configured. Environment: {'Kubernetes' if in_kubernetes else 'Development/Testing'}")
+        logging.info(f"Logs directory: {logs_dir}")
+    except Exception as e:
+        print(f"Error configuring logging: {str(e)}")
+        print("Gedetailleerde foutinformatie:")
+        import traceback
+        traceback.print_exc()
+        # Fall back to basic configuration
+        logging.basicConfig(level=logging.INFO)
+
+
 
 LOGGING = {
     'version': 1,
@@ -290,7 +285,7 @@ LOGGING = {
         'file_app': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_app.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_app.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -298,7 +293,7 @@ LOGGING = {
         'file_workers': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_workers.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_workers.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -306,7 +301,7 @@ LOGGING = {
         'file_chat_client': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_chat_client.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_chat_client.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -314,7 +309,7 @@ LOGGING = {
         'file_chat_workers': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_chat_workers.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_chat_workers.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -322,7 +317,7 @@ LOGGING = {
         'file_api': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_api.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_api.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -330,7 +325,7 @@ LOGGING = {
         'file_beat': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_beat.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_beat.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -338,7 +333,7 @@ LOGGING = {
         'file_entitlements': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/eveai_entitlements.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'eveai_entitlements.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -346,7 +341,7 @@ LOGGING = {
         'file_sqlalchemy': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/sqlalchemy.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'sqlalchemy.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -354,7 +349,7 @@ LOGGING = {
         'file_security': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/security.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'security.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -362,7 +357,7 @@ LOGGING = {
         'file_rag_tuning': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/rag_tuning.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'rag_tuning.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -370,7 +365,7 @@ LOGGING = {
         'file_embed_tuning': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/embed_tuning.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'embed_tuning.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -378,7 +373,7 @@ LOGGING = {
         'file_business_events': {
             'level': 'INFO',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/business_events.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'business_events.log'),
             'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 2,
             'formatter': 'standard',
@@ -388,98 +383,102 @@ LOGGING = {
             'level': 'DEBUG',
             'formatter': 'standard',
         },
+        'json_console': {
+            'class': 'logging.StreamHandler',
+            'level': 'INFO',
+            'formatter': 'json',
+            'stream': 'ext://sys.stdout',
+        },
         'tuning_file': {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
-            'filename': 'logs/tuning.log',
+            'filename': os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), 'logs', 'tuning.log'),
             'maxBytes': 1024 * 1024 * 3,  # 3MB
             'backupCount': 3,
             'formatter': 'tuning',
         },
-        'graylog': {
-            'level': 'DEBUG',
-            'class': 'graypy.GELFUDPHandler',
-            'host': GRAYLOG_HOST,
-            'port': GRAYLOG_PORT,
-            'debugging_fields': True,
-            'formatter': 'graylog'
-        },
     },
     'formatters': {
         'standard': {
             'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d]: %(message)s',
             'datefmt': '%Y-%m-%d %H:%M:%S'
         },
-        'graylog': {
-            'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
-                      '[Thread: %(threadName)s]: %(message)s',
-            'datefmt': '%Y-%m-%d %H:%M:%S',
-            '()': GraylogFormatter
-        },
         'tuning': {
             '()': TuningFormatter,
             'datefmt': '%Y-%m-%d %H:%M:%S UTC'
+        },
+        'json': {
+            'format': '%(message)s',
+            'class': 'logging.Formatter' if not 'pythonjsonlogger' in sys.modules else 'pythonjsonlogger.jsonlogger.JsonFormatter',
+            'json_default': lambda obj: str(obj) if isinstance(obj, (dt, Exception)) else None,
+            'json_ensure_ascii': False,
+            'rename_fields': {
+                'asctime': 'timestamp',
+                'levelname': 'severity'
+            },
+            'timestamp': True,
+            'datefmt': '%Y-%m-%dT%H:%M:%S.%fZ'
         }
     },
     'loggers': {
         'eveai_app': {  # logger for the eveai_app
-            'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ],
+            'handlers': ['file_app'],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_workers': {  # logger for the eveai_workers
-            'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ],
+            'handlers': ['file_workers'],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_chat_client': {  # logger for the eveai_chat
-            'handlers': ['file_chat_client', 'graylog', ] if env == 'production' else ['file_chat_client', ],
+            'handlers': ['file_chat_client'],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_chat_workers': {  # logger for the eveai_chat_workers
-            'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ],
+            'handlers': ['file_chat_workers'],
             'level': 'DEBUG',
             'propagate': False
         },
-        'eveai_api': {  # logger for the eveai_chat_workers
+        'eveai_api': {  # logger for the eveai_api
-            'handlers': ['file_api', 'graylog', ] if env == 'production' else ['file_api', ],
+            'handlers': ['file_api'],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_beat': {  # logger for the eveai_beat
-            'handlers': ['file_beat', 'graylog', ] if env == 'production' else ['file_beat', ],
+            'handlers': ['file_beat'],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_entitlements': {  # logger for the eveai_entitlements
-            'handlers': ['file_entitlements', 'graylog', ] if env == 'production' else ['file_entitlements', ],
+            'handlers': ['file_entitlements'],
             'level': 'DEBUG',
             'propagate': False
         },
         'sqlalchemy.engine': {  # logger for the sqlalchemy
-            'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ],
+            'handlers': ['file_sqlalchemy'],
             'level': 'DEBUG',
             'propagate': False
         },
         'security': {  # logger for the security
-            'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ],
+            'handlers': ['file_security'],
             'level': 'DEBUG',
             'propagate': False
         },
         'business_events': {
-            'handlers': ['file_business_events', 'graylog'],
+            'handlers': ['file_business_events'],
             'level': 'DEBUG',
             'propagate': False
         },
         # Single tuning logger
         'tuning': {
-            'handlers': ['tuning_file', 'graylog'] if env == 'production' else ['tuning_file'],
+            'handlers': ['tuning_file'],
             'level': 'DEBUG',
             'propagate': False,
         },
         '': {  # root logger
-            'handlers': ['console'],
+            'handlers': ['console'] if os.environ.get('KUBERNETES_SERVICE_HOST') is None else ['json_console'],
             'level': 'WARNING',  # Set higher level for root to minimize noise
             'propagate': False
         },

config/processors/globals/AUTOMAGIC_HTML_PROCESSOR/1.0.0.yaml

@@ -0,0 +1,14 @@
+version: "1.0.0"
+name: "HTML Processor"
+file_types: "html"
+description: "A processor for HTML files, driven by AI"
+configuration:
+  custom_instructions:
+    name: "Custom Instructions"
+    description: "Some custom instruction to guide our AI agent in parsing your HTML file"
+    type: "text"
+    required: false
+metadata:
+  author: "Josako"
+  date_added: "2025-06-25"
+  description: "A processor for HTML files, driven by AI"

config/prompts/globals/automagic_html_parse/1.0.0.yaml

@@ -0,0 +1,30 @@
+version: "1.0.0"
+content: |
+  You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The 
+  generated files will be used to generate embeddings in a RAG-system.
+  
+  # Best practices are:
+  - Respect wordings and language(s) used in the HTML.
+  - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
+  - Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list.  
+  - Be careful of encoding of the text. Everything needs to be human readable.
+
+  You only return relevant information, and filter out non-relevant information, such as:
+  - information found in menu bars, sidebars, footers or headers
+  - information in forms, buttons
+  
+  Process the file or text carefully, and take a stepped approach. The resulting markdown should be the result of the 
+  processing of the complete input html file. Answer with the pure markdown, without any other text.
+  
+  {custom_instructions}
+  
+  HTML to be processed is in between triple backquotes.
+  
+  ```{html}```
+
+llm_model: "mistral.mistral-small-latest"
+metadata:
+  author: "Josako"
+  date_added: "2025-06-25"
+  description: "An aid in transforming HTML-based inputs to markdown, fully automatic"
+  changes: "Initial version"

config/prompts/globals/translation_with_context/1.0.0.yaml

@@ -0,0 +1,15 @@
+version: "1.0.0"
+content: >
+  You are a top translator. We need you to translate {text_to_translate} into {target_language}, taking into account 
+  this context:
+  
+  {context}
+  
+  I only want you to return the translation. No explanation, no options. I need to be able to directly use your answer
+  without further interpretation. If more than one option is available, present me with the most probable one.
+llm_model: "mistral.ministral-8b-latest"
+metadata:
+  author: "Josako"
+  date_added: "2025-06-23"
+  description: "An assistant to translate given a context."
+  changes: "Initial version"

config/prompts/globals/translation_without_context/1.0.0.yaml

@@ -0,0 +1,12 @@
+version: "1.0.0"
+content: >
+  You are a top translator. We need you to translate {text_to_translate} into {target_language}.
+  
+  I only want you to return the translation. No explanation, no options. I need to be able to directly use your answer
+  without further interpretation. If more than one option is available, present me with the most probable one.
+llm_model: "mistral.ministral-8b-latest"
+metadata:
+  author: "Josako"
+  date_added: "2025-06-23"
+  description: "An assistant to translate without context."
+  changes: "Initial version"

config/specialists/traicie/TRAICIE_SELECTION_SPECIALIST/1.2.0.yaml

@@ -1,4 +1,4 @@
-version: "1.1.0"
+version: "1.3.0"
 name: "Traicie Selection Specialist"
 framework: "crewai"
 partner: "traicie"
@@ -108,13 +108,13 @@ results:
     description: "List of vacancy competencies and their descriptions"
     required: false
 agents:
-  - type: "TRAICIE_HR_BP_AGENT"
+  - type: "TRAICIE_RECRUITER"
     version: "1.0"
 tasks:
-  - type: "TRAICIE_GET_COMPETENCIES_TASK"
+  - type: "TRAICIE_KO_CRITERIA_INTERVIEW_DEFINITION"
-    version: "1.1"
+    version: "1.0"
 metadata:
   author: "Josako"
-  date_added: "2025-05-27"
+  date_added: "2025-06-16"
-  changes: "Add make to the selection specialist"
+  changes: "Realising the actual interaction with the LLM"
   description: "Assistant to create a new Vacancy based on Vacancy Text"

config/specialists/traicie/TRAICIE_SELECTION_SPECIALIST/1.3.0.yaml

@@ -0,0 +1,120 @@
+version: "1.3.0"
+name: "Traicie Selection Specialist"
+framework: "crewai"
+partner: "traicie"
+chat: false
+configuration:
+  name:
+    name: "Name"
+    description: "The name the specialist is called upon."
+    type: "str"
+    required: true
+  role_reference:
+    name: "Role Reference"
+    description: "A customer reference to the role"
+    type: "str"
+    required: false
+  make:
+    name: "Make"
+    description: "The make for which the role is defined and the selection specialist is created"
+    type: "system"
+    system_name: "tenant_make"
+    required: true
+  competencies:
+    name: "Competencies"
+    description: "An ordered list of competencies."
+    type: "ordered_list"
+    list_type: "competency_details"
+    required: true
+  tone_of_voice:
+    name: "Tone of Voice"
+    description: "The tone of voice the specialist uses to communicate"
+    type: "enum"
+    allowed_values: ["Professional & Neutral", "Warm & Empathetic", "Energetic & Enthusiastic", "Accessible & Informal", "Expert & Trustworthy", "No-nonsense & Goal-driven"]
+    default: "Professional & Neutral"
+    required: true
+  language_level:
+    name: "Language Level"
+    description: "Language level to be used when communicating, relating to CEFR levels"
+    type: "enum"
+    allowed_values: ["Basic", "Standard", "Professional"]
+    default: "Standard"
+    required: true
+  welcome_message:
+    name: "Welcome Message"
+    description: "Introductory text given by the specialist - but translated according to Tone of Voice, Language Level and Starting Language"
+    type: "text"
+    required: false
+  closing_message:
+    name: "Closing Message"
+    description: "Closing message given by the specialist - but translated according to Tone of Voice, Language Level and Starting Language"
+    type: "text"
+    required: false
+competency_details:
+  title:
+    name: "Title"
+    description: "Competency Title"
+    type: "str"
+    required: true
+  description:
+    name: "Description"
+    description: "Description (in context of the role) of the competency"
+    type: "text"
+    required: true
+  is_knockout:
+    name: "KO"
+    description: "Defines if the competency is a knock-out criterium"
+    type: "boolean"
+    required: true
+    default: false
+  assess:
+    name: "Assess"
+    description: "Indication if this competency is to be assessed"
+    type: "boolean"
+    required: true
+    default: true
+arguments:
+  region:
+    name: "Region"
+    type: "str"
+    description: "The region of the specific vacancy"
+    required: false
+  working_schedule:
+    name: "Work Schedule"
+    type: "str"
+    description: "The work schedule or employment type of the specific vacancy"
+    required: false
+  start_date:
+    name: "Start Date"
+    type: "date"
+    description: "The start date of the specific vacancy"
+    required: false
+  language:
+    name: "Language"
+    type: "str"
+    description: "The language (2-letter code) used to start the conversation"
+    required: true
+  interaction_mode:
+    name: "Interaction Mode"
+    type: "enum"
+    description: "The interaction mode the specialist will start working in."
+    allowed_values: ["Job Application", "Seduction"]
+    default: "Job Application"
+    required: true
+results:
+  competencies:
+    name: "competencies"
+    type: "List[str, str]"
+    description: "List of vacancy competencies and their descriptions"
+    required: false
+agents:
+  - type: "TRAICIE_RECRUITER_AGENT"
+    version: "1.0"
+tasks:
+  - type: "TRAICIE_KO_CRITERIA_INTERVIEW_DEFINITION_TASK"
+    version: "1.0"
+metadata:
+  author: "Josako"
+  date_added: "2025-06-18"
+  changes: "Add make to the selection specialist"
+  description: "Assistant to create a new Vacancy based on Vacancy Text"

config/tasks/traicie/TRAICIE_KO_CRITERIA_INTERVIEW_DEFINITION_TASK/1.0.0.yaml

@@ -5,17 +5,24 @@ task_description: >
   (both description and title). The criteria are in between triple backquotes.You need to prepare for the interviews, 
   and are to provide for each of these ko criteria:
   
-  - A question to ask the recruitment candidate describing the context of the ko criterium. Use your experience to not 
+  - A short question to ask the recruitment candidate describing the context of the ko criterium. Use your experience to
-  just ask a closed question, but a question from which you can indirectly derive a positive or negative qualification 
+  ask a question that enables us to verify compliancy to the criterium.
-  of the criterium based on the answer of the candidate.
+  - A set of 2 short answers to that question, from the candidates perspective. One of the answers will result in a 
-  - A set of max 5 answers on that question, from the candidates perspective. One of the answers will result in a 
+  positive evaluation of the criterium, the other one in a negative evaluation. Mark each of the answers as positive
-  positive evaluation of the criterium, the other ones in a negative evaluation. Mark each of the answers as positive
   or negative. 
   Describe the answers from the perspective of the candidate. Be sure to include all necessary aspects in you answers. 
   
   Apply the following tone of voice in both questions and answers: {tone_of_voice}
+  Use the following description to understand tone of voice:
+  
+  {tone_of_voice_context}
+  
   Apply the following language level in both questions and answers: {language_level}
+  
   Use {language} as language for both questions and answers.
+  Use the following description to understand language_level:
+  
+  {language_level_context}
   
   ```{ko_criteria}```
   
@@ -25,7 +32,8 @@ expected_output: >
   For each of the ko criteria, you provide:
   - the exact title as specified in the original language
   - the question in {language}
-  - a set of answers, with for each answer an indication if it is the correct answer, or a false response. In {language}.
+  - a positive answer, resulting in a positive evaluation of the criterium. In {language}.
+  - a negative answer, resulting in a negative evaluation of the criterium. In {language}.
   {custom_expected_output}
 metadata:
   author: "Josako"

config/tasks/traicie/TRAICIE_KO_CRITERIA_INTERVIEW_DEFINITION_TASK/1.0.1.yaml

@@ -0,0 +1,37 @@
+version: "1.0.0"
+name: "KO Criteria Interview Definition"
+task_description: >
+  In context of a vacancy in your company {tenant_name}, you are provided with a set of competencies 
+  (both description and title). The competencies are in between triple backquotes. The competencies provided should be
+  handled as knock-out criteria.
+  For each of the knock-out criteria, you need to define
+  
+  - A short (1 sentence), closed-ended question (Yes / No) to ask the recruitment candidate. Use your experience to ask a question that 
+  enables us to verify compliancy to the criterium.
+  - A set of 2 short answers (1 small sentence each) to that question (positive answer / negative answer), from the 
+  candidates perspective. 
+  The positive answer will result in a positive evaluation of the criterium, the negative answer in a negative evaluation
+  of the criterium. Try to avoid just using Yes / No as positive and negative answers.
+  
+  Apply the following tone of voice in both questions and answers: {tone_of_voice}, i.e. {tone_of_voice_context}
+
+  Apply the following language level in both questions and answers: {language_level}, i.e. {language_level_context}
+  
+  Use {language} as language for both questions and answers.
+  
+  ```{ko_criteria}```
+  
+  {custom_description}
+
+expected_output: >
+  For each of the ko criteria, you provide:
+  - the exact title as specified in the original language
+  - the question in {language}
+  - a positive answer, resulting in a positive evaluation of the criterium. In {language}.
+  - a negative answer, resulting in a negative evaluation of the criterium. In {language}.
+  {custom_expected_output}
+metadata:
+  author: "Josako"
+  date_added: "2025-06-20"
+  description: "A Task to define interview Q&A from given KO Criteria"
+  changes: "Improvement to ensure closed-ended questions and short descriptions"

config/type_defs/agent_types.py

@@ -32,5 +32,10 @@ AGENT_TYPES = {
         "name": "Traicie HR BP Agent",
         "description": "An HR Business Partner Agent",
         "partner": "traicie"
-    }
+    },
+    "TRAICIE_RECRUITER_AGENT": {
+        "name": "Traicie Recruiter Agent",
+        "description": "An Senior Recruiter Agent",
+        "partner": "traicie"
+    },
 }

config/type_defs/processor_types.py

@@ -24,5 +24,10 @@ PROCESSOR_TYPES = {
         "name": "DOCX Processor",
         "description": "A processor for DOCX files",
         "file_types": "docx",
-    }
+    },
+    "AUTOMAGIC_HTML_PROCESSOR": {
+        "name": "AutoMagic HTML Processor",
+        "description": "A processor for HTML files, driven by AI",
+        "file_types": "html, htm",
+    },
 }

config/type_defs/prompt_types.py

@@ -28,4 +28,12 @@ PROMPT_TYPES = {
         "name": "transcript",
         "description": "An assistant to transform a transcript to markdown.",
     },
+    "translation_with_context": {
+        "name": "translation_with_context",
+        "description": "An assistant to translate text with context",
+    },
+    "translation_without_context": {
+        "name": "translation_without_context",
+        "description": "An assistant to translate text without context",
+    },
 }

config/type_defs/task_types.py

@@ -41,5 +41,10 @@ TASK_TYPES = {
         "name": "Traicie Get KO Criteria",
         "description": "A Task to get KO Criteria from a Vacancy Text",
         "partner": "traicie"
+    },
+    "TRAICIE_KO_CRITERIA_INTERVIEW_DEFINITION_TASK": {
+        "name": "Traicie KO Criteria Interview Definition",
+        "description": "A Task to define KO Criteria questions to be used during the interview",
+        "partner": "traicie"
     }
 }

content/changelog/1.0/1.0.0.md

@@ -5,10 +5,50 @@ All notable changes to EveAI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.3.8-alfa]
+
+### Added
+- Translation Service
+- Automagic HTML Processor 
+- Allowed languages defined at level of Tenant Make
+
+### Changed
+- For changes in existing functionality.
+- Allow to activate / de-activate Processors
+- Align all document views with session catalog
+- Allow different processor types to handle the same file types
+- Remove welcome message from tenant_make customisation, add to specialist configuration
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- Adapt TRAICIE_ROLE_DEFINITION_SPECIALIST to latest requirements
+- Allow for empty historical messages
+- Ensure client can cope with empty customisation options
+- Ensure only tenant-defined makes are selectable throughout the application
+- Refresh partner info when adding Partner Services
+
+### Security
+- In case of vulnerabilities.
+
+## [2.3.7-alfa]
+
+### Added
+- Basic Base Specialist additions for handling phases and transferring data between state and output
+- Introduction of URL and QR-code for MagicLink
+
+### Changed
+- Logging improvement & simplification (remove Graylog)
+- Traicie Selection Specialist v1.3 - full roundtrip & full process
+
 ## [2.3.6-alfa]
 
 ### Added
-- Full Chat Client functionaltiy, including Forms, ESS, theming
+- Full Chat Client functionality, including Forms, ESS, theming
 - First Demo version of Traicie Selection Specialist
 
 ## [2.3.5-alfa]

docker/compose_dev.yaml

@@ -24,7 +24,7 @@ x-common-variables: &common-variables
   FLOWER_PASSWORD: 'Jungles'
   OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7'
   GROQ_API_KEY: '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'
-  MISTRAL_API_KEY: 'jGDc6fkCbt0iOC0jQsbuZhcjLWBPGc2b'
+  MISTRAL_API_KEY: '0f4ZiQ1kIpgIKTHX8d0a8GOD2vAgVqEn'
   ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2'
   JWT_SECRET_KEY: 'bsdMkmQ8ObfMD52yAFg4trrvjgjMhuIqg2fjDpD/JqvgY0ccCcmlsEnVFmR79WPiLKEA3i8a5zmejwLZKl4v9Q=='
   API_ENCRYPTION_KEY: 'xfF5369IsredSrlrYZqkM9ZNrfUASYYS6TCcAR9UKj4='

docker/compose_test.yaml

@@ -26,7 +26,7 @@ x-common-variables: &common-variables
   REDIS_PORT: '6379'
   FLOWER_USER: 'Felucia'
   FLOWER_PASSWORD: 'Jungles'
-  MISTRAL_API_KEY: 'Vkwgr67vUs6ScKmcFF2QVw7uHKgq0WEN'
+  MISTRAL_API_KEY: 'qunKSaeOkFfLteNiUO77RCsXXSLK65Ec'
   JWT_SECRET_KEY: '7e9c8b3a215f4d6e90712c5d8f3b97a60e482c15f39a7d68bcd45910ef23a784'
   API_ENCRYPTION_KEY: 'kJ7N9p3IstyRGkluYTryM8ZMnfUBSXWR3TCfDG9VLc4='
   MINIO_ENDPOINT: minio:9000

eveai_api/__init__.py

@@ -12,7 +12,7 @@ import logging.config
 from common.models.user import TenantDomain
 from common.utils.cors_utils import get_allowed_origins
 from common.utils.database import Database
-from config.logging_config import LOGGING
+from config.logging_config import configure_logging
 from .api.document_api import document_ns
 from .api.auth import auth_ns
 from .api.specialist_execution_api import specialist_execution_ns
@@ -40,7 +40,7 @@ def create_app(config_file=None):
     app.celery = make_celery(app.name, app.config)
     init_celery(app.celery, app)
 
-    logging.config.dictConfig(LOGGING)
+    configure_logging()
     logger = logging.getLogger(__name__)
 
     logger.info("eveai_api starting up")

eveai_app/__init__.py

@@ -13,7 +13,7 @@ import common.models.interaction
 import common.models.entitlements
 import common.models.document
 from common.utils.startup_eveai import perform_startup_actions
-from config.logging_config import LOGGING
+from config.logging_config import configure_logging
 from common.utils.security import set_tenant_session_data
 from common.utils.errors import register_error_handlers
 from common.utils.celery_utils import make_celery, init_celery
@@ -47,8 +47,16 @@ def create_app(config_file=None):
     except OSError:
         pass
 
-    logging.config.dictConfig(LOGGING)
+    # Configureer logging op basis van de omgeving (K8s of traditioneel)
+    try:
+        configure_logging()
         logger = logging.getLogger(__name__)
+        # Test dat logging werkt
+        logger.debug("Logging test in eveai_app")
+    except Exception as e:
+        print(f"Critical Error Initialising Error: {str(e)}")
+        import traceback
+        traceback.print_exc()
 
     logger.info("eveai_app starting up")
 
@@ -92,6 +100,45 @@ def create_app(config_file=None):
     #     app.logger.debug(f"Before request - Session data: {session}")
     #     app.logger.debug(f"Before request - Request headers: {request.headers}")
 
+    @app.before_request
+    def before_request():
+        from flask import session, request
+        from flask_login import current_user
+        import datetime
+
+        app.logger.debug(f"Before request - URL: {request.url}")
+        app.logger.debug(f"Before request - Session permanent: {session.permanent}")
+
+        # Log session expiry tijd als deze bestaat
+        if current_user.is_authenticated:
+            # Controleer of sessie permanent is (nodig voor PERMANENT_SESSION_LIFETIME)
+            if not session.permanent:
+                session.permanent = True
+                app.logger.debug("Session marked as permanent (enables 60min timeout)")
+
+            # Log wanneer sessie zou verlopen
+            if '_permanent' in session:
+                expires_at = datetime.datetime.now() + app.permanent_session_lifetime
+                app.logger.debug(f"Session will expire at: {expires_at} (60 min from now)")
+
+    @app.route('/debug/session')
+    def debug_session():
+        from flask import session
+        from flask_security import current_user
+        import datetime
+
+        if current_user.is_authenticated:
+            info = {
+                'session_permanent': session.permanent,
+                'session_lifetime_minutes': app.permanent_session_lifetime.total_seconds() / 60,
+                'session_refresh_enabled': app.config.get('SESSION_REFRESH_EACH_REQUEST'),
+                'current_time': datetime.datetime.now().isoformat(),
+                'session_data_keys': list(session.keys())
+            }
+            return jsonify(info)
+        else:
+            return jsonify({'error': 'Not authenticated'})
+
     # Register template filters
     register_filters(app)
 
@@ -154,8 +201,3 @@ def register_cache_handlers(app):
     register_specialist_cache_handlers(cache_manager)
     from common.utils.cache.license_cache import register_license_cache_handlers
     register_license_cache_handlers(cache_manager)
-
-
-
-
-

eveai_app/templates/document/document_versions.html

@@ -4,13 +4,13 @@
 {% block title %}Document Versions{% endblock %}
 
 {% block content_title %}Document Versions{% endblock %}
-{% block content_description %}View Versions for {{ document }}{% endblock %}
+{% block content_description %}View Versions for Document <b>{{ document }}</b>{% endblock %}
 {% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
 
 {% block content %}
 <div class="container">
     <form method="POST" action="{{ url_for('document_bp.handle_document_version_selection') }}" id="documentVersionsForm">
-        {{ render_selectable_table(headers=["ID", "URL", "Object Name", "File Type", "Process.", "Proces. Start", "Proces. Finish", "Proces. Error"], rows=rows, selectable=True, id="versionsTable") }}
+        {{ render_selectable_table(headers=["ID", "File Type", "File Size", "Process.", "Proces. Start", "Proces. Finish", "Proces. Error"], rows=rows, selectable=True, id="versionsTable") }}
         <div class="form-group mt-3 d-flex justify-content-between">
             <div>
                 <button type="submit" name="action" value="edit_document_version" class="btn btn-primary" onclick="return validateTableSelection('documentVersionsForm')">Edit Document Version</button>

eveai_app/templates/document/documents.html

@@ -4,14 +4,13 @@
 {% block title %}Documents{% endblock %}
 
 {% block content_title %}Documents{% endblock %}
-{% block content_description %}View Documents for Tenant{% endblock %}
+{% block content_description %}View Documents for Catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 {% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
 
 {% block content %}
     <!-- Filter Form -->
     {% set filter_form %}
         <form method="GET" action="{{ url_for('document_bp.documents') }}">
-            {{ render_filter_field('catalog_id', 'Catalog', filter_options['catalog_id'], filters.get('catalog_id', [])) }}
             {{ render_filter_field('validity', 'Validity', filter_options['validity'], filters.get('validity', [])) }}
 
             <button type="submit" class="btn btn-primary">Apply Filters</button>
@@ -27,7 +26,6 @@
                 headers=[
                     {"text": "ID", "sort": "id"},
                     {"text": "Name", "sort": "name"},
-                    {"text": "Catalog", "sort": "catalog_name"},
                     {"text": "Valid From", "sort": "valid_from"},
                     {"text": "Valid To", "sort": "valid_to"}
                 ],

eveai_app/templates/document/edit_processor.html

@@ -4,7 +4,7 @@
 {% block title %}Edit Processor{% endblock %}
 
 {% block content_title %}Edit Processor{% endblock %}
-{% block content_description %}Edit a Processor (for a Catalog){% endblock %}
+{% block content_description %}Edit Processor for Catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 
 {% block content %}
     <form method="post">

eveai_app/templates/document/edit_retriever.html

@@ -4,7 +4,7 @@
 {% block title %}Edit Retriever{% endblock %}
 
 {% block content_title %}Edit Retriever{% endblock %}
-{% block content_description %}Edit a Retriever (for a Catalog){% endblock %}
+{% block content_description %}Edit a Retriever for catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 
 {% block content %}
     <form method="post">

eveai_app/templates/document/processor.html

@@ -4,7 +4,7 @@
 {% block title %}Processor Registration{% endblock %}
 
 {% block content_title %}Register Processor{% endblock %}
-{% block content_description %}Define a new processor (for a catalog){% endblock %}
+{% block content_description %}Define a new processor for Catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 
 {% block content %}
     <form method="post">

eveai_app/templates/document/processors.html

@@ -4,13 +4,13 @@
 {% block title %}Processors{% endblock %}
 
 {% block content_title %}Processors{% endblock %}
-{% block content_description %}View Processors for Tenant{% endblock %}
+{% block content_description %}View Processors for Catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 {% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
 
 {% block content %}
 <div class="container">
     <form method="POST" action="{{ url_for('document_bp.handle_processor_selection') }}" id="processorsForm">
-        {{ render_selectable_table(headers=["Processor ID", "Name", "Type", "Catalog ID"], rows=rows, selectable=True, id="retrieversTable") }}
+        {{ render_selectable_table(headers=["Processor ID", "Name", "Type", "Active"], rows=rows, selectable=True, id="retrieversTable") }}
         <div class="form-group mt-3 d-flex justify-content-between">
             <div>
                 <button type="submit" name="action" value="edit_processor" class="btn btn-primary" onclick="return validateTableSelection('processorsForm')">Edit Processor</button>

eveai_app/templates/document/retriever.html

@@ -4,7 +4,7 @@
 {% block title %}Retriever Registration{% endblock %}
 
 {% block content_title %}Register Retriever{% endblock %}
-{% block content_description %}Define a new retriever (for a catalog){% endblock %}
+{% block content_description %}Define a new retriever for Catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 
 {% block content %}
     <form method="post">

eveai_app/templates/document/retrievers.html

@@ -4,13 +4,13 @@
 {% block title %}Retrievers{% endblock %}
 
 {% block content_title %}Retrievers{% endblock %}
-{% block content_description %}View Retrievers for Tenant{% endblock %}
+{% block content_description %}View Retrievers for Catalog <b>{% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}</b>{% endblock %}
 {% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
 
 {% block content %}
 <div class="container">
     <form method="POST" action="{{ url_for('document_bp.handle_retriever_selection') }}" id="retrieversForm">
-        {{ render_selectable_table(headers=["Retriever ID", "Name", "Type", "Catalog ID"], rows=rows, selectable=True, id="retrieversTable") }}
+        {{ render_selectable_table(headers=["Retriever ID", "Name", "Type"], rows=rows, selectable=True, id="retrieversTable") }}
         <div class="form-group mt-3 d-flex justify-content-between">
             <div>
                 <button type="submit" name="action" value="edit_retriever" class="btn btn-primary" onclick="return validateTableSelection('retrieversForm')">Edit Retriever</button>

eveai_app/templates/interaction/edit_specialist_magic_link.html

@@ -9,11 +9,30 @@
 {% block content %}
     <form method="post">
         {{ form.hidden_tag() }}
-        {% set disabled_fields = ['magic_link_code'] %}
+        {% set disabled_fields = ['magic_link_code', 'chat_client_url', 'qr_code_url'] %}
         {% set exclude_fields = [] %}
         <!-- Render Static Fields -->
         {% for field in form.get_static_fields() %}
+            {% if field.name == 'qr_code_url' and field.data %}
+                <div class="form-group">
+                    <label for="{{ field.id }}">{{ field.label.text }}</label>
+                    <div style="max-width: 200px;">
+                        <img src="{{ field.data }}" alt="QR Code" class="img-fluid">
+                    </div>
+                    <input type="hidden" name="{{ field.name }}" value="{{ field.data|e }}">
+                </div>
+            {% elif field.name == 'chat_client_url' %}
+                <div class="form-group">
+                    <label for="{{ field.id }}" class="form-label">{{ field.label.text }}</label>
+                    <div class="input-group">
+                        <input type="text" class="form-control" value="{{ field.data }}" id="{{ field.id }}" readonly>
+                        <a href="{{ field.data }}" class="btn btn-primary" target="_blank">Open link</a>
+                    </div>
+                    <input type="hidden" name="{{ field.name }}" value="{{ field.data|e }}">
+                </div>
+            {% else %}
                 {{ render_field(field, disabled_fields, exclude_fields) }}
+            {% endif %}
         {% endfor %}
         <!-- Render Dynamic Fields -->
         {% for collection_name, fields in form.get_dynamic_fields().items() %}

eveai_app/views/document_forms.py

@@ -71,15 +71,6 @@ class ProcessorForm(FlaskForm):
     name = StringField('Name', validators=[DataRequired(), Length(max=50)])
     description = TextAreaField('Description', validators=[Optional()])
 
-    # Catalog for the Retriever
-    catalog = QuerySelectField(
-        'Catalog ID',
-        query_factory=lambda: Catalog.query.all(),
-        allow_blank=True,
-        get_label='name',
-        validators=[DataRequired()],
-    )
-
     # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
     type = SelectField('Processor Type', validators=[DataRequired()])
 
@@ -89,6 +80,7 @@ class ProcessorForm(FlaskForm):
                                   default=2000)
     max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
                                   default=3000)
+    active = BooleanField('Active', default=True)
     tuning = BooleanField('Enable Embedding Tuning', default=False)
 
     # Metadata fields
@@ -108,14 +100,6 @@ class EditProcessorForm(DynamicFormBase):
     name = StringField('Name', validators=[DataRequired(), Length(max=50)])
     description = TextAreaField('Description', validators=[Optional()])
 
-    # Catalog for the Retriever
-    catalog = QuerySelectField(
-        'Catalog ID',
-        query_factory=lambda: Catalog.query.all(),
-        allow_blank=True,
-        get_label='name',
-        validators=[Optional()],
-    )
     type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True})
 
     sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
@@ -124,6 +108,7 @@ class EditProcessorForm(DynamicFormBase):
                                   default=2000)
     max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
                                   default=3000)
+    active = BooleanField('Active', default=True)
     tuning = BooleanField('Enable Embedding Tuning', default=False)
 
     # Metadata fields
@@ -134,14 +119,7 @@ class EditProcessorForm(DynamicFormBase):
 class RetrieverForm(FlaskForm):
     name = StringField('Name', validators=[DataRequired(), Length(max=50)])
     description = TextAreaField('Description', validators=[Optional()])
-    # Catalog for the Retriever
+
-    catalog = QuerySelectField(
-        'Catalog ID',
-        query_factory=lambda: Catalog.query.all(),
-        allow_blank=True,
-        get_label='name',
-        validators=[Optional()],
-    )
     # Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config)
     type = SelectField('Retriever Type', validators=[DataRequired()])
     tuning = BooleanField('Enable Tuning', default=False)
@@ -160,14 +138,7 @@ class RetrieverForm(FlaskForm):
 class EditRetrieverForm(DynamicFormBase):
     name = StringField('Name', validators=[DataRequired(), Length(max=50)])
     description = TextAreaField('Description', validators=[Optional()])
-    # Catalog for the Retriever
+
-    catalog = QuerySelectField(
-        'Catalog ID',
-        query_factory=lambda: Catalog.query.all(),
-        allow_blank=True,
-        get_label='name',
-        validators=[Optional()],
-    )
     # Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config)
     type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True})
     tuning = BooleanField('Enable Tuning', default=False)

eveai_app/views/document_list_view.py

@@ -1,5 +1,5 @@
-from datetime import datetime
+from datetime import datetime as dt, timezone as tz
-from flask import request, render_template, session
+from flask import request, render_template, session, current_app
 from sqlalchemy import desc, asc, or_, and_, cast, Integer
 from common.models.document import Document, Catalog
 from common.utils.filtered_list_view import FilteredListView
@@ -7,31 +7,19 @@ from common.utils.view_assistants import prepare_table_for_macro
 
 
 class DocumentListView(FilteredListView):
-    allowed_filters = ['catalog_id', 'validity']
+    allowed_filters = ['validity']
-    allowed_sorts = ['id', 'name', 'catalog_name', 'valid_from', 'valid_to']
+    allowed_sorts = ['id', 'name', 'valid_from', 'valid_to']
 
     def get_query(self):
-        return Document.query.join(Catalog).add_columns(
+        catalog_id = session.get('catalog_id')
-            Document.id,
+        current_app.logger.debug(f"Catalog ID: {catalog_id}")
-            Document.name,
+        return Document.query.filter_by(catalog_id=catalog_id)
-            Catalog.name.label('catalog_name'),
-            Document.valid_from,
-            Document.valid_to
-        )
 
     def apply_filters(self, query):
         filters = request.args.to_dict(flat=False)
 
-        if 'catalog_id' in filters:
-            catalog_ids = filters['catalog_id']
-            if catalog_ids:
-                # Convert catalog_ids to a list of integers
-                catalog_ids = [int(cid) for cid in catalog_ids if cid.isdigit()]
-                if catalog_ids:
-                    query = query.filter(Document.catalog_id.in_(catalog_ids))
-
         if 'validity' in filters:
-            now = datetime.utcnow().date()
+            now = dt.now(tz.utc).date()
             if 'valid' in filters['validity']:
                 query = query.filter(
                     and_(
@@ -47,9 +35,6 @@ class DocumentListView(FilteredListView):
         sort_order = request.args.get('sort_order', 'asc')
 
         if sort_by in self.allowed_sorts:
-            if sort_by == 'catalog_name':
-                column = Catalog.name
-            else:
             column = getattr(Document, sort_by)
 
             if sort_order == 'asc':
@@ -61,42 +46,39 @@ class DocumentListView(FilteredListView):
 
     def get(self):
         query = self.get_query()
-        query = self.apply_filters(query)
+        # query = self.apply_filters(query)
-        query = self.apply_sorting(query)
+        # query = self.apply_sorting(query)
         pagination = self.paginate(query)
 
         def format_date(date):
-            if isinstance(date, datetime):
+            if isinstance(date, dt):
                 return date.strftime('%Y-%m-%d')
             elif isinstance(date, str):
                 return date
             else:
                 return ''
 
+        current_app.logger.debug(f"Items retrieved: {pagination.items}")
         rows = [
             [
                 {'value': item.id, 'class': '', 'type': 'text'},
                 {'value': item.name, 'class': '', 'type': 'text'},
-                {'value': item.catalog_name, 'class': '', 'type': 'text'},
                 {'value': format_date(item.valid_from), 'class': '', 'type': 'text'},
                 {'value': format_date(item.valid_to), 'class': '', 'type': 'text'}
             ] for item in pagination.items
         ]
 
-        catalogs = Catalog.query.all()
-
         context = {
             'rows': rows,
             'pagination': pagination,
             'filters': request.args.to_dict(flat=False),
             'sort_by': request.args.get('sort_by', 'id'),
             'sort_order': request.args.get('sort_order', 'asc'),
-            'filter_options': self.get_filter_options(catalogs)
+            'filter_options': self.get_filter_options()
         }
         return render_template(self.template, **context)
 
-    def get_filter_options(self, catalogs):
+    def get_filter_options(self):
         return {
-            'catalog_id': [(str(cat.id), cat.name) for cat in catalogs],
             'validity': [('valid', 'Valid'), ('all', 'All')]
         }

eveai_app/views/document_views.py

@@ -16,7 +16,7 @@ from common.extensions import db, cache_manager, minio_client
 from common.models.interaction import Specialist, SpecialistRetriever
 from common.utils.document_utils import create_document_stack, start_embedding_task, process_url, \
     edit_document, \
-    edit_document_version, refresh_document, clean_url
+    edit_document_version, refresh_document, clean_url, is_file_type_supported_by_catalog
 from common.utils.dynamic_field_utils import create_default_config_from_type_config
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException, EveAIException
@@ -110,7 +110,6 @@ def handle_catalog_selection():
         current_app.logger.info(f'Setting session catalog to {catalog.name}')
         session['catalog_id'] = catalog_id
         session['catalog_name'] = catalog.name
-        current_app.logger.info(f'Finished setting session catalog to {catalog.name}')
     elif action == 'edit_catalog':
         return redirect(prefixed_url_for('document_bp.edit_catalog', catalog_id=catalog_id))
 
@@ -157,7 +156,7 @@ def processor():
         tenant_id = session.get('tenant').get('id')
         new_processor = Processor()
         form.populate_obj(new_processor)
-        new_processor.catalog_id = form.catalog.data.id
+        new_processor.catalog_id = session.get('catalog_id')
         processor_config = cache_manager.processors_config_cache.get_config(new_processor.type)
         new_processor.configuration = create_default_config_from_type_config(
             processor_config["configuration"])
@@ -204,9 +203,6 @@ def edit_processor(processor_id):
         form.populate_obj(processor)
         processor.configuration = form.get_dynamic_data('configuration')
 
-        # Update catalog relationship
-        processor.catalog_id = form.catalog.data.id if form.catalog.data else None
-
         # Update logging information
         update_logging_information(processor, dt.now(tz.utc))
 
@@ -235,14 +231,19 @@ def processors():
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
 
-    query = Processor.query.order_by(Processor.id)
+    catalog_id = session.get('catalog_id', None)
+    if not catalog_id:
+        flash('You need to set a Session Catalog before adding Documents or URLs', 'warning')
+        return redirect(prefixed_url_for('document_bp.catalogs'))
+
+    query = Processor.query.filter_by(catalog_id=catalog_id).order_by(Processor.id)
 
     pagination = query.paginate(page=page, per_page=per_page)
     the_processors = pagination.items
 
     # prepare table data
     rows = prepare_table_for_macro(the_processors,
-                                   [('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')])
+                                   [('id', ''), ('name', ''), ('type', ''), ('active', '')])
 
     # Render the catalogs in a template
     return render_template('document/processors.html', rows=rows, pagination=pagination)
@@ -272,7 +273,7 @@ def retriever():
         tenant_id = session.get('tenant').get('id')
         new_retriever = Retriever()
         form.populate_obj(new_retriever)
-        new_retriever.catalog_id = form.catalog.data.id
+        new_retriever.catalog_id = session.get('catalog_id')
         new_retriever.type_version = cache_manager.retrievers_version_tree_cache.get_latest_version(
             new_retriever.type)
 
@@ -301,12 +302,6 @@ def edit_retriever(retriever_id):
     # Get the retriever or return 404
     retriever = Retriever.query.get_or_404(retriever_id)
 
-    if retriever.catalog_id:
-        # If catalog_id is just an ID, fetch the Catalog object
-        retriever.catalog = Catalog.query.get(retriever.catalog_id)
-    else:
-        retriever.catalog = None
-
     # Create form instance with the retriever
     form = EditRetrieverForm(request.form, obj=retriever)
 
@@ -319,9 +314,6 @@ def edit_retriever(retriever_id):
         form.populate_obj(retriever)
         retriever.configuration = form.get_dynamic_data('configuration')
 
-        # Update catalog relationship
-        retriever.catalog_id = form.catalog.data.id if form.catalog.data else None
-
         # Update logging information
         update_logging_information(retriever, dt.now(tz.utc))
 
@@ -350,14 +342,19 @@ def retrievers():
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
 
-    query = Retriever.query.order_by(Retriever.id)
+    catalog_id = session.get('catalog_id', None)
+    if not catalog_id:
+        flash('You need to set a Session Catalog before adding Documents or URLs', 'warning')
+        return redirect(prefixed_url_for('document_bp.catalogs'))
+
+    query = Retriever.query.filter_by(catalog_id=catalog_id).order_by(Retriever.id)
 
     pagination = query.paginate(page=page, per_page=per_page)
     the_retrievers = pagination.items
 
     # prepare table data
     rows = prepare_table_for_macro(the_retrievers,
-                                   [('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')])
+                                   [('id', ''), ('name', ''), ('type', '')])
 
     # Render the catalogs in a template
     return render_template('document/retrievers.html', rows=rows, pagination=pagination)
@@ -400,6 +397,8 @@ def add_document():
             filename = secure_filename(file.filename)
             extension = filename.rsplit('.', 1)[1].lower()
 
+            is_file_type_supported_by_catalog(catalog_id, extension)
+
             catalog_properties = form.get_dynamic_data("tagging_fields")
 
             api_input = {
@@ -451,6 +450,8 @@ def add_url():
 
             file_content, filename, extension = process_url(url, tenant_id)
 
+            is_file_type_supported_by_catalog(catalog_id, extension)
+
             catalog_properties = {}
             full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
             document_version_configurations = full_config['document_version_configurations']
@@ -489,6 +490,11 @@ def add_url():
 @document_bp.route('/documents', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
 def documents():
+    catalog_id = session.get('catalog_id', None)
+    if not catalog_id:
+        flash('You need to set a Session Catalog before adding Documents or URLs', 'warning')
+        return redirect(prefixed_url_for('document_bp.catalogs'))
+
     view = DocumentListView(Document, 'document/documents.html', per_page=10)
     return view.get()
 
@@ -609,7 +615,7 @@ def edit_document_version_view(document_version_id):
 @roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
 def document_versions(document_id):
     doc = Document.query.get_or_404(document_id)
-    doc_desc = f'Document {doc.name}'
+    doc_desc = f'{doc.name}'
 
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
@@ -621,8 +627,7 @@ def document_versions(document_id):
     pagination = query.paginate(page=page, per_page=per_page, error_out=False)
     doc_langs = pagination.items
 
-    rows = prepare_table_for_macro(doc_langs, [('id', ''), ('url', ''),
+    rows = prepare_table_for_macro(doc_langs, [('id', ''), ('file_type', ''), ('file_size', ''),
-                                               ('object_name', ''), ('file_type', ''),
                                                ('processing', ''), ('processing_started_at', ''),
                                                ('processing_finished_at', ''), ('processing_error', '')])
 

eveai_app/views/dynamic_form_base.py

@@ -312,7 +312,7 @@ class DynamicFormBase(FlaskForm):
                 field_class = SelectField
                 tenant_id = session.get('tenant').get('id')
                 makes = TenantMake.query.filter_by(tenant_id=tenant_id).all()
-                choices = [(make.name, make.name) for make in makes]
+                choices = [(make.id, make.name) for make in makes]
                 extra_classes = ''
                 field_kwargs = {'choices': choices}
 
@@ -328,6 +328,16 @@ class DynamicFormBase(FlaskForm):
             initial_data: Optional initial data for the fields
         """
         current_app.logger.debug(f"Adding dynamic fields for collection {collection_name} with config: {config}")
+
+        if isinstance(initial_data, str):
+            try:
+                initial_data = json.loads(initial_data)
+            except (json.JSONDecodeError, TypeError):
+                current_app.logger.error(f"Invalid JSON in initial_data: {initial_data}")
+                initial_data = {}
+        elif initial_data is None:
+            initial_data = {}
+
         # Store the full configuration for later use in get_list_type_configs_js
         if not hasattr(self, '_full_configs'):
             self._full_configs = {}
@@ -581,7 +591,10 @@ class DynamicFormBase(FlaskForm):
                 except Exception as e:
                     current_app.logger.error(f"Error converting initial data to patterns: {e}")
             elif isinstance(field, DateField):
+                if field.data:
                     data[original_field_name] = field.data.isoformat()
+                else:
+                    data[original_field_name] = None
             else:
                 data[original_field_name] = field.data
         return data

eveai_app/views/entitlements_views.py

@@ -259,6 +259,10 @@ def view_usages():
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
 
+    if not session.get('tenant', None):
+        flash('You can only view usage for a Tenant. Select a Tenant to continue!', 'danger')
+        return redirect(prefixed_url_for('user_bp.select_tenant'))
+
     tenant_id = session.get('tenant').get('id')
     query = LicenseUsage.query.filter_by(tenant_id=tenant_id).order_by(desc(LicenseUsage.id))
 

eveai_app/views/interaction_forms.py

@@ -1,3 +1,4 @@
+from flask import session
 from flask_wtf import FlaskForm
 from wtforms import (StringField, BooleanField, SelectField, TextAreaField)
 from wtforms.fields.datetime import DateField
@@ -162,6 +163,8 @@ class EditSpecialistMagicLinkForm(DynamicFormBase):
                                   render_kw={'readonly': True})
     specialist_id = IntegerField('Specialist', validators=[DataRequired()], render_kw={'readonly': True})
     specialist_name = StringField('Specialist Name', validators=[DataRequired()], render_kw={'readonly': True})
+    chat_client_url = StringField('Chat Client URL', validators=[Optional()], render_kw={'readonly': True})
+    qr_code_url = StringField('QR Code', validators=[Optional()], render_kw={'readonly': True})
     tenant_make_id = SelectField('Tenant Make', validators=[Optional()], coerce=int)
     valid_from = DateField('Valid From', id='form-control datepicker', validators=[Optional()])
     valid_to = DateField('Valid To', id='form-control datepicker', validators=[Optional()])
@@ -179,7 +182,8 @@ class EditSpecialistMagicLinkForm(DynamicFormBase):
             self.specialist_name.data = ''
 
         # Dynamically populate the tenant_make field with None as first option
-        tenant_makes = TenantMake.query.all()
+        tenant_id = session.get('tenant').get('id')
+        tenant_makes = TenantMake.query.filter_by(tenant_id=tenant_id).all()
         self.tenant_make_id.choices = [(0, 'None')] + [(make.id, make.name) for make in tenant_makes]
 
 

eveai_app/views/interaction_views.py

@@ -702,12 +702,13 @@ def specialist_magic_link():
             new_spec_ml_tenant.tenant_id = tenant_id
 
             # Define the make valid for this magic link
-            make_id = SpecialistServices.get_specialist_system_field(new_specialist_magic_link.specialist_id,
+            specialist = Specialist.query.get(new_specialist_magic_link.specialist_id)
-                                                                     "make", "tenant_make")
+            make_id = specialist.configuration.get('make', None)
+            current_app.logger.debug(f"make_id defined in specialist: {make_id}")
             if make_id:
-                new_spec_ml_tenant.tenant_make_id = make_id
+                new_specialist_magic_link.tenant_make_id = make_id
             elif session.get('tenant').get('default_tenant_make_id'):
-                new_spec_ml_tenant.tenant_make_id = session.get('tenant').get('default_tenant_make_id')
+                new_specialist_magic_link.tenant_make_id = session.get('tenant').get('default_tenant_make_id')
 
             db.session.add(new_specialist_magic_link)
             db.session.add(new_spec_ml_tenant)
@@ -748,6 +749,56 @@ def edit_specialist_magic_link(specialist_magic_link_id):
         else:
             form.tenant_make_id.data = specialist_ml.tenant_make_id
 
+        # Set the chat client URL
+        tenant_id = session.get('tenant').get('id')
+        chat_client_prefix = current_app.config.get('CHAT_CLIENT_PREFIX', 'chat_client/chat/')
+        base_url = request.url_root
+        magic_link_code = specialist_ml.magic_link_code
+
+        # Parse the URL om poortinformatie te behouden als deze afwijkt van de standaard
+        url_parts = request.url.split('/')
+        host_port = url_parts[2]  # Dit bevat zowel hostname als poort indien aanwezig
+
+        # Generate the full URL for chat client with magic link code
+        chat_client_url = f"{request.scheme}://{host_port}/{chat_client_prefix}{magic_link_code}"
+        form.chat_client_url.data = chat_client_url
+
+        # Generate QR code as data URI for direct embedding in HTML
+        try:
+            import qrcode
+            import io
+            import base64
+
+            # Generate QR code as PNG for better compatibility
+            qr = qrcode.QRCode(
+                version=1,
+                error_correction=qrcode.constants.ERROR_CORRECT_L,
+                box_size=10,
+                border=4
+            )
+            qr.add_data(chat_client_url)
+            qr.make(fit=True)
+
+            # Generate PNG image in memory
+            img = qr.make_image(fill_color="black", back_color="white")
+            buffer = io.BytesIO()
+            img.save(buffer, format='PNG')
+            img_data = buffer.getvalue()
+
+            # Create data URI for direct embedding in HTML
+            img_base64 = base64.b64encode(img_data).decode('utf-8')
+            data_uri = f"data:image/png;base64,{img_base64}"
+
+            # Store the data URI in the form data
+            form.qr_code_url.data = data_uri
+
+            current_app.logger.debug(f"QR code generated successfully for {magic_link_code}")
+            current_app.logger.debug(f"QR code data URI starts with: {data_uri[:50]}...")
+
+        except Exception as e:
+            current_app.logger.error(f"Failed to generate QR code: {str(e)}")
+            form.qr_code_url.data = "Error generating QR code"
+
     if form.validate_on_submit():
         # Update the basic fields
         form.populate_obj(specialist_ml)

eveai_app/views/partner_views.py

@@ -62,6 +62,7 @@ def edit_partner(partner_id):
         update_logging_information(partner, dt.now(tz.utc))
         db.session.commit()
         flash('Partner updated successfully.', 'success')
+        refresh_session_partner(partner.id)
         return redirect(
             prefixed_url_for('partner_bp.edit_partner',
                              partner_id=partner.id))  # Assuming there's a user profile view to redirect to
@@ -197,6 +198,7 @@ def edit_partner_service(partner_service_id):
             db.session.commit()
             flash('Partner Service updated successfully.', 'success')
             current_app.logger.info(f"Partner Service {partner_service.name} updated successfully! ")
+            refresh_session_partner(partner_id)
         except SQLAlchemyError as e:
             db.session.rollback()
             flash(f'Failed to update Partner Service: {str(e)}', 'danger')
@@ -339,4 +341,7 @@ def add_partner_service_for_tenant(partner_service_id):
         return redirect(prefixed_url_for('partner_bp.partner_services'))
 
 
-
+def refresh_session_partner(partner_id):
+    if session.get('partner', None):
+        if partner_id == session['partner']['id']:
+            session['partner'] = Partner.query.get_or_404(partner_id).to_dict()

eveai_app/views/user_forms.py

@@ -196,6 +196,14 @@ class TenantMakeForm(DynamicFormBase):
     active = BooleanField('Active', validators=[Optional()], default=True)
     website = StringField('Website', validators=[DataRequired(), Length(max=255)])
     logo_url = StringField('Logo URL', validators=[Optional(), Length(max=255)])
+    allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[Optional()])
+
+    def __init__(self, *args, **kwargs):
+        super(TenantMakeForm, self).__init__(*args, **kwargs)
+        # Initialiseer de taalopties met taalcodes en vlaggen
+        lang_details = current_app.config['SUPPORTED_LANGUAGE_DETAILS']
+        self.allowed_languages.choices = [(details['iso 639-1'], f"{details['flag']} {details['iso 639-1']}") 
+                                         for name, details in lang_details.items()]
 
 class EditTenantMakeForm(DynamicFormBase):
     id = IntegerField('ID', widget=HiddenInput())
@@ -204,6 +212,14 @@ class EditTenantMakeForm(DynamicFormBase):
     active = BooleanField('Active', validators=[Optional()], default=True)
     website = StringField('Website', validators=[DataRequired(), Length(max=255)])
     logo_url = StringField('Logo URL', validators=[Optional(), Length(max=255)])
+    allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[Optional()])
+
+    def __init__(self, *args, **kwargs):
+        super(EditTenantMakeForm, self).__init__(*args, **kwargs)
+        # Initialiseer de taalopties met taalcodes en vlaggen
+        lang_details = current_app.config['SUPPORTED_LANGUAGE_DETAILS']
+        self.allowed_languages.choices = [(details['iso 639-1'], f"{details['flag']} {details['iso 639-1']}") 
+                                         for name, details in lang_details.items()]
 
 
 

eveai_app/views/user_views.py

@@ -655,6 +655,8 @@ def tenant_make():
         new_tenant_make.tenant_id = tenant_id
         customisation_options = form.get_dynamic_data("configuration")
         new_tenant_make.chat_customisation_options = json.dumps(customisation_options)
+        # Verwerk allowed_languages als array
+        new_tenant_make.allowed_languages = form.allowed_languages.data if form.allowed_languages.data else None
         set_logging_information(new_tenant_make, dt.now(tz.utc))
 
         try:
@@ -703,6 +705,10 @@ def edit_tenant_make(tenant_make_id):
     # Create form instance with the tenant make
     form = EditTenantMakeForm(request.form, obj=tenant_make)
 
+    # Initialiseer de allowed_languages selectie met huidige waarden
+    if tenant_make.allowed_languages:
+        form.allowed_languages.data = tenant_make.allowed_languages
+
     customisation_config = cache_manager.customisations_config_cache.get_config("CHAT_CLIENT_CUSTOMISATION")
     form.add_dynamic_fields("configuration", customisation_config, tenant_make.chat_customisation_options)
 
@@ -710,6 +716,8 @@ def edit_tenant_make(tenant_make_id):
         # Update basic fields
         form.populate_obj(tenant_make)
         tenant_make.chat_customisation_options = form.get_dynamic_data("configuration")
+        # Verwerk allowed_languages als array
+        tenant_make.allowed_languages = form.allowed_languages.data if form.allowed_languages.data else None
 
         # Update logging information
         update_logging_information(tenant_make, dt.now(tz.utc))

eveai_beat/__init__.py

@@ -4,7 +4,7 @@ from flask import Flask
 import os
 
 from common.utils.celery_utils import make_celery, init_celery
-from config.logging_config import LOGGING
+from config.logging_config import configure_logging
 from config.config import get_config
 
 
@@ -21,7 +21,7 @@ def create_app(config_file=None):
         case _:
             app.config.from_object(get_config('dev'))
 
-    logging.config.dictConfig(LOGGING)
+    configure_logging()
 
     register_extensions(app)
 

eveai_chat_client/__init__.py

@@ -9,7 +9,7 @@ from common.extensions import (db, bootstrap, cors, csrf, session,
                               minio_client, simple_encryption, metrics, cache_manager, content_manager)
 from common.models.user import Tenant, SpecialistMagicLinkTenant
 from common.utils.startup_eveai import perform_startup_actions
-from config.logging_config import LOGGING
+from config.logging_config import configure_logging
 from eveai_chat_client.utils.errors import register_error_handlers
 from common.utils.celery_utils import make_celery, init_celery
 from common.utils.template_filters import register_filters
@@ -39,7 +39,7 @@ def create_app(config_file=None):
     except OSError:
         pass
 
-    logging.config.dictConfig(LOGGING)
+    configure_logging()
     logger = logging.getLogger(__name__)
 
     logger.info("eveai_chat_client starting up")

eveai_chat_client/static/assets/js/chat-app.js

@@ -131,11 +131,13 @@ export const ChatApp = {
             const historicalMessages = chatConfig.messages || [];
 
             if (historicalMessages.length > 0) {
-                this.allMessages = historicalMessages.map(msg => {
+                this.allMessages = historicalMessages
+                    .filter(msg => msg !== null && msg !== undefined) // Filter null/undefined berichten uit
+                    .map(msg => {
                         // Zorg voor een correct geformatteerde bericht-object
                         return {
                             id: this.messageIdCounter++,
-                        content: typeof msg === 'string' ? msg : msg.content || '',
+                            content: typeof msg === 'string' ? msg : (msg.content || ''),
                             sender: msg.sender || 'ai',
                             type: msg.type || 'text',
                             timestamp: msg.timestamp || new Date().toISOString(),

eveai_chat_client/views/chat_views.py

@@ -86,7 +86,13 @@ def chat(magic_link_code):
         session['chat_session_id'] = SpecialistServices.start_session()
 
         # Get customisation options with defaults
+        current_app.logger.debug(f"Make Customisation Options: {tenant_make.chat_customisation_options}")
+        try:
             customisation = get_default_chat_customisation(tenant_make.chat_customisation_options)
+        except Exception as e:
+            current_app.logger.error(f"Error processing customisation options: {str(e)}")
+            # Fallback to default customisation
+            customisation = get_default_chat_customisation(None)
 
         # Start a new chat session
         session['chat_session_id'] = SpecialistServices.start_session()

eveai_chat_workers/__init__.py

@@ -5,7 +5,7 @@ import os
 
 from common.utils.celery_utils import make_celery, init_celery
 from common.extensions import db, cache_manager
-from config.logging_config import LOGGING
+from config.logging_config import configure_logging
 from config.config import get_config
 
 
@@ -22,7 +22,7 @@ def create_app(config_file=None):
         case _:
             app.config.from_object(get_config('dev'))
 
-    logging.config.dictConfig(LOGGING)
+    configure_logging()
 
     app.logger.info('Starting up eveai_chat_workers...')
     register_extensions(app)

eveai_chat_workers/definitions/language_level/language_level_v1_0.py

@@ -0,0 +1,20 @@
+LANGUAGE_LEVEL = [
+  {
+    "name": "Basic",
+    "description": "Short, simple sentences. Minimal jargon. Lots of visual and concrete language.",
+    "cefr_level": "A2 - B1",
+    "ideal_audience": "Manual laborers, entry-level roles, newcomers with another native language"
+  },
+  {
+    "name": "Standard",
+    "description": "Clear spoken language. Well-formulated without difficult words.",
+    "cefr_level": "B2",
+    "ideal_audience": "Retail, administration, logistics, early-career professionals"
+  },
+  {
+    "name": "Professional",
+    "description": "Business language with technical terms where needed. More complex sentence structures.",
+    "cefr_level": "C1",
+    "ideal_audience": "Management, HR, technical profiles"
+  }
+]

eveai_chat_workers/definitions/tone_of_voice/tone_of_voice_v1_0.py

@@ -0,0 +1,32 @@
+TONE_OF_VOICE = [
+  {
+    "name": "Professional & Neutral",
+    "description": "Business-like, clear, to the point. Focused on facts.",
+    "when_to_use": "Corporate jobs, legal roles, formal sectors"
+  },
+  {
+    "name": "Warm & Empathetic",
+    "description": "Human, compassionate, reassuring.",
+    "when_to_use": "Healthcare, education, HR, social professions"
+  },
+  {
+    "name": "Energetic & Enthusiastic",
+    "description": "Upbeat, persuasive, motivating.",
+    "when_to_use": "Sales, marketing, hospitality, start-ups"
+  },
+  {
+    "name": "Accessible & Informal",
+    "description": "Casual, approachable, friendly, and human.",
+    "when_to_use": "Youth-focused, entry-level, retail, creative sectors"
+  },
+  {
+    "name": "Expert & Trustworthy",
+    "description": "Calm authority, advisory tone, knowledgeable.",
+    "when_to_use": "IT, engineering, consultancy, medical profiles"
+  },
+  {
+    "name": "No-nonsense & Goal-driven",
+    "description": "Direct, efficient, pragmatic.",
+    "when_to_use": "Technical, logistics, blue-collar jobs, production environments"
+  }
+]

eveai_chat_workers/outputs/traicie/knockout_questions/knockout_questions_v1_0.py

@@ -0,0 +1,15 @@
+from typing import List, Optional
+from pydantic import BaseModel, Field
+from eveai_chat_workers.outputs.globals.basic_types.list_item import ListItem
+
+class KOQuestion(BaseModel):
+    title: str = Field(..., description="The title of the knockout criterium.")
+    question: str = Field(..., description="The corresponding question asked to the candidate.")
+    answer_positive: Optional[str] = Field(None, description="The answer to the question, resulting in a positive outcome.")
+    answer_negative: Optional[str] = Field(None, description="The answer to the question, resulting in a negative outcome.")
+
+class KOQuestions(BaseModel):
+    ko_questions: List[KOQuestion] = Field(
+        default_factory=list,
+        description="KO Questions and answers."
+    )

eveai_chat_workers/specialists/base_specialist.py

@@ -4,7 +4,8 @@ from typing import Dict, Any, List
 from flask import current_app
 
 from common.extensions import cache_manager
-from common.models.interaction import SpecialistRetriever
+from common.models.interaction import SpecialistRetriever, Specialist
+from common.models.user import Tenant
 from common.utils.execution_progress import ExecutionProgressTracker
 from config.logging_config import TuningLogger
 from eveai_chat_workers.retrievers.base import BaseRetriever
@@ -17,7 +18,9 @@ class BaseSpecialistExecutor(ABC):
 
     def __init__(self, tenant_id: int, specialist_id: int, session_id: str, task_id: str):
         self.tenant_id = tenant_id
+        self.tenant = Tenant.query.get_or_404(tenant_id)
         self.specialist_id = specialist_id
+        self.specialist = Specialist.query.get_or_404(specialist_id)
         self.session_id = session_id
         self.task_id = task_id
         self.tuning = False
@@ -96,6 +99,37 @@ class BaseSpecialistExecutor(ABC):
     def update_progress(self, processing_type, data) -> None:
         self.ept.send_update(self.task_id, processing_type, data)
 
+    def _replace_system_variables(self, text: str) -> str:
+        """
+        Replace all system variables in the text with their corresponding values.
+        System variables are in the format 'tenant_<attribute_name>'
+
+        Args:
+            text: The text containing system variables to replace
+
+        Returns:
+            str: The text with all system variables replaced
+        """
+        if not text:
+            return text
+
+        from common.utils.model_utils import replace_variable_in_template
+
+        # Find all tenant_* variables and replace them with tenant attribute values
+        # Format of variables: tenant_name, tenant_code, etc.
+        result = text
+
+        # Get all attributes of the tenant object
+        tenant_attrs = vars(self.tenant)
+
+        # Replace all tenant_* variables
+        for attr_name, attr_value in tenant_attrs.items():
+            variable = f"tenant_{attr_name}"
+            if variable in result:
+                result = replace_variable_in_template(result, variable, str(attr_value))
+
+        return result
+
     @abstractmethod
     def execute_specialist(self, arguments: SpecialistArguments) -> SpecialistResult:
         """Execute the specialist's logic"""

eveai_chat_workers/specialists/crewai_base_specialist.py

@@ -33,10 +33,6 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
     def __init__(self, tenant_id: int, specialist_id: int, session_id: str, task_id):
         super().__init__(tenant_id, specialist_id, session_id, task_id)
 
-        # Check and load the specialist
-        self.specialist = Specialist.query.get_or_404(specialist_id)
-        # Set the specific configuration for the SPIN Specialist
-        # self.specialist_configuration = json.loads(self.specialist.configuration)
         self.tuning = self.specialist.tuning
         # Initialize retrievers
         self.retrievers = self._initialize_retrievers()
@@ -54,15 +50,20 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
         self._task_pydantic_outputs: Dict[str, Type[BaseModel]] = {}
         self._task_state_names: Dict[str, str] = {}
 
-        # Processed configurations
+        # State-Result relations (for adding / restoring information to / from history
+        self._state_result_relations: Dict[str, str] = {}
+
+        # Process configurations
         self._config = cache_manager.crewai_processed_config_cache.get_specialist_config(tenant_id, specialist_id)
         self._config_task_agents()
         self._config_pydantic_outputs()
         self._instantiate_crew_assets()
         self._instantiate_specialist()
+        self._config_state_result_relations()
 
         # Retrieve history
         self._cached_session = cache_manager.chat_session_cache.get_cached_session(self.session_id)
+        self._restore_state_from_history()
         # Format history for the prompt
         self._formatted_history = self._generate_formatted_history()
 
@@ -110,6 +111,19 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
         """Configure the task pydantic outputs by adding task-output combinations. Use _add_pydantic_output()"""
         raise NotImplementedError
 
+    def _add_state_result_relation(self, state_name: str, result_name: str = None):
+        """Add a state-result relation to the specialist. This is used to add information to the history
+           If result_name is None, the state name is used as the result name. (default behavior)
+        """
+        if not result_name:
+            result_name = state_name
+        self._state_result_relations[state_name] = result_name
+
+    @abstractmethod
+    def _config_state_result_relations(self):
+        """Configure the state-result relations by adding state-result combinations. Use _add_state_result_relation()"""
+        raise NotImplementedError
+
     @property
     def task_pydantic_outputs(self):
         return self._task_pydantic_outputs
@@ -127,7 +141,9 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
         for agent in self.specialist.agents:
             agent_config = cache_manager.agents_config_cache.get_config(agent.type, agent.type_version)
             agent_role = agent_config.get('role', '').replace('{custom_role}', agent.role or '')
+            agent_role = self._replace_system_variables(agent_role)
             agent_goal = agent_config.get('goal', '').replace('{custom_goal}', agent.goal or '')
+            agent_goal = self._replace_system_variables(agent_goal)
             agent_backstory = agent_config.get('backstory', '').replace('{custom_backstory}', agent.backstory or '')
             agent_full_model_name = agent_config.get('full_model_name', 'mistral.mistral-large-latest')
             agent_temperature = agent_config.get('temperature', 0.3)
@@ -152,6 +168,7 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
             task_config = cache_manager.tasks_config_cache.get_config(task.type, task.type_version)
             task_description = (task_config.get('task_description', '')
                                 .replace('{custom_description}', task.task_description or ''))
+            task_description = self._replace_system_variables(task_description)
             task_expected_output = (task_config.get('expected_output', '')
                                     .replace('{custom_expected_output}', task.expected_output or ''))
             # dynamically build the arguments
@@ -161,9 +178,12 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
                 "verbose": task.tuning
             }
             task_name = task.type.lower()
+            current_app.logger.debug(f"Task {task_name} is getting processed")
             if task_name in self._task_pydantic_outputs:
                 task_kwargs["output_pydantic"] = self._task_pydantic_outputs[task_name]
+                current_app.logger.debug(f"Task {task_name} has an output pydantic: {self._task_pydantic_outputs[task_name]}")
             if task_name in self._task_agents:
+                current_app.logger.debug(f"Task {task_name} has an agent: {self._task_agents[task_name]}")
                 task_kwargs["agent"] = self._agents[self._task_agents[task_name]]
 
             # Instantiate the task with dynamic arguments
@@ -328,6 +348,27 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
 
         return formatted_context, citations
 
+    def _update_specialist_results(self, specialist_results: SpecialistResult) -> SpecialistResult:
+        """Update the specialist results with the latest state information"""
+        update_data = {}
+        state_dict = self.flow.state.model_dump()
+        for state_name, result_name in self._state_result_relations.items():
+            if state_name in state_dict and state_dict[state_name] is not None:
+                update_data[result_name] = state_dict[state_name]
+
+        return specialist_results.model_copy(update=update_data)
+
+    def _restore_state_from_history(self):
+        """Restore the state from the history"""
+        if not self._cached_session.interactions:
+            return
+        last_interaction = self._cached_session.interactions[-1]
+        if not last_interaction.specialist_results:
+            return
+        for state_name, result_name in self._state_result_relations.items():
+            if result_name in last_interaction.specialist_results:
+                setattr(self.flow.state, state_name, last_interaction.specialist_results[result_name])
+
     @abstractmethod
     def execute(self, arguments: SpecialistArguments, formatted_context: str, citations: List[int]) -> SpecialistResult:
         raise NotImplementedError
@@ -354,8 +395,10 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor):
                 "detailed_query": detailed_query,
                 "citations": citations,
             }
-            final_result = result.model_copy(update=modified_result)
+            intermediate_result = result.model_copy(update=modified_result)
         else:
-            final_result = self.execute(arguments, "", [])
+            intermediate_result = self.execute(arguments, "", [])
+
+        final_result = self._update_specialist_results(intermediate_result)
 
         return final_result

eveai_chat_workers/specialists/traicie/TRAICIE_ROLE_DEFINITION_SPECIALIST/1_3.py

@@ -18,6 +18,9 @@ from eveai_chat_workers.outputs.traicie.competencies.competencies_v1_1 import Co
 from eveai_chat_workers.specialists.crewai_base_classes import EveAICrewAICrew, EveAICrewAIFlow, EveAIFlowState
 from common.services.interaction.specialist_services import SpecialistServices
 
+NEW_SPECIALIST_TYPE = "TRAICIE_SELECTION_SPECIALIST"
+NEW_SPECIALIST_TYPE_VERSION = "1.3"
+
 
 class SpecialistExecutor(CrewAIBaseSpecialistExecutor):
     """
@@ -66,6 +69,9 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor):
             self.role_definition_crew
         )
 
+    def _config_state_result_relations(self):
+        pass
+
     def execute(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
         self.log_tuning("Traicie Role Definition Specialist execution started", {})
 
@@ -117,8 +123,8 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor):
         new_specialist = Specialist(
             name=name,
             description=f"Specialist for {arguments.role_name} role",
-            type="TRAICIE_SELECTION_SPECIALIST",
+            type=NEW_SPECIALIST_TYPE,
-            type_version="1.1",
+            type_version=NEW_SPECIALIST_TYPE_VERSION,
             tuning=False,
             configuration=selection_config,
         )
@@ -130,7 +136,7 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor):
             current_app.logger.error(f"Error creating selection specialist: {str(e)}")
             raise e
 
-        SpecialistServices.initialize_specialist(new_specialist.id, "TRAICIE_SELECTION_SPECIALIST", "1.0")
+        SpecialistServices.initialize_specialist(new_specialist.id, NEW_SPECIALIST_TYPE, NEW_SPECIALIST_TYPE_VERSION)
 
 
 

eveai_chat_workers/specialists/traicie/TRAICIE_SELECTION_SPECIALIST/1_3.py

@@ -0,0 +1,350 @@
+import asyncio
+import json
+from os import wait
+from typing import Optional, List, Dict, Any
+from datetime import date
+from time import sleep
+from crewai.flow.flow import start, listen, and_
+from flask import current_app
+from pydantic import BaseModel, Field, EmailStr
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.extensions import db
+from common.models.user import Tenant
+from common.models.interaction import Specialist
+from eveai_chat_workers.outputs.globals.basic_types.list_item import ListItem
+from eveai_chat_workers.outputs.traicie.knockout_questions.knockout_questions_v1_0 import KOQuestions, KOQuestion
+from eveai_chat_workers.specialists.crewai_base_specialist import CrewAIBaseSpecialistExecutor
+from eveai_chat_workers.specialists.specialist_typing import SpecialistResult, SpecialistArguments
+from eveai_chat_workers.outputs.traicie.competencies.competencies_v1_1 import Competencies
+from eveai_chat_workers.specialists.crewai_base_classes import EveAICrewAICrew, EveAICrewAIFlow, EveAIFlowState
+from common.services.interaction.specialist_services import SpecialistServices
+from common.extensions import cache_manager
+from eveai_chat_workers.definitions.language_level.language_level_v1_0 import LANGUAGE_LEVEL
+from eveai_chat_workers.definitions.tone_of_voice.tone_of_voice_v1_0 import TONE_OF_VOICE
+from common.utils.eveai_exceptions import EveAISpecialistExecutionError
+
+
+class SpecialistExecutor(CrewAIBaseSpecialistExecutor):
+    """
+    type: TRAICIE_SELECTION_SPECIALIST
+    type_version: 1.1
+    Traicie Selection Specialist Executor class
+    """
+
+    def __init__(self, tenant_id, specialist_id, session_id, task_id, **kwargs):
+        self.role_definition_crew = None
+
+        super().__init__(tenant_id, specialist_id, session_id, task_id)
+
+        # Load the Tenant & set language
+        self.tenant = Tenant.query.get_or_404(tenant_id)
+
+    @property
+    def type(self) -> str:
+        return "TRAICIE_SELECTION_SPECIALIST"
+
+    @property
+    def type_version(self) -> str:
+        return "1.3"
+
+    def _config_task_agents(self):
+        self._add_task_agent("traicie_ko_criteria_interview_definition_task", "traicie_recruiter_agent")
+
+    def _config_pydantic_outputs(self):
+        self._add_pydantic_output("traicie_ko_criteria_interview_definition_task", KOQuestions, "ko_questions")
+
+    def _config_state_result_relations(self):
+        self._add_state_result_relation("ko_criteria_questions")
+        self._add_state_result_relation("ko_criteria_scores")
+        self._add_state_result_relation("competency_questions")
+        self._add_state_result_relation("competency_scores")
+        self._add_state_result_relation("personal_contact_data")
+
+    def _instantiate_specialist(self):
+        verbose = self.tuning
+
+        ko_def_agents = [self.traicie_recruiter_agent]
+        ko_def_tasks = [self.traicie_ko_criteria_interview_definition_task]
+        self.ko_def_crew = EveAICrewAICrew(
+            self,
+            "KO Criteria Interview Definition Crew",
+            agents=ko_def_agents,
+            tasks=ko_def_tasks,
+            verbose=verbose,
+        )
+
+        self.flow = SelectionFlow(
+            self,
+            self.ko_def_crew
+        )
+
+    def execute(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("Traicie Selection Specialist execution started", {})
+
+        current_app.logger.debug(f"Arguments: {arguments.model_dump()}")
+        current_app.logger.debug(f"Formatted Context: {formatted_context}")
+        current_app.logger.debug(f"Formatted History: {self._formatted_history}")
+        current_app.logger.debug(f"Cached Chat Session: {self._cached_session}")
+
+        if not self._cached_session.interactions:
+            specialist_phase = "initial"
+        else:
+            specialist_phase = self._cached_session.interactions[-1].specialist_results.get('phase', 'initial')
+
+        results = None
+
+        match specialist_phase:
+            case "initial":
+                results = self.execute_initial_state(arguments, formatted_context, citations)
+            case "ko_question_evaluation":
+                results = self.execute_ko_question_evaluation(arguments, formatted_context, citations)
+            case "personal_contact_data":
+                results = self.execute_personal_contact_data(arguments, formatted_context, citations)
+            case "no_valid_candidate":
+                results = self.execute_no_valid_candidate(arguments, formatted_context, citations)
+            case "candidate_selected":
+                results = self.execute_candidate_selected(arguments, formatted_context, citations)
+
+        self.log_tuning(f"Traicie Selection Specialist execution ended", {"Results": results.model_dump() if results else "No info"})
+
+        return results
+
+
+    def execute_initial_state(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("Traicie Selection Specialist initial_state_execution started", {})
+
+        current_app.logger.debug(f"Specialist Competencies:\n{self.specialist.configuration.get("competencies", [])}")
+
+        ko_competencies = []
+        for competency in self.specialist.configuration.get("competencies", []):
+            if competency["is_knockout"] is True and competency["assess"] is True:
+                current_app.logger.debug(f"Assessable Knockout competency: {competency}")
+                ko_competencies.append({"title: ": competency["title"], "description": competency["description"]})
+
+        tone_of_voice = self.specialist.configuration.get('tone_of_voice', 'Professional & Neutral')
+        selected_tone_of_voice = next(
+            (item for item in TONE_OF_VOICE if item["name"] == tone_of_voice),
+            None  # fallback indien niet gevonden
+        )
+        current_app.logger.debug(f"Selected tone of voice: {selected_tone_of_voice}")
+        tone_of_voice_context = f"{selected_tone_of_voice["description"]}"
+
+        language_level = self.specialist.configuration.get('language_level', 'Standard')
+        selected_language_level = next(
+            (item for item in LANGUAGE_LEVEL if item["name"] == language_level),
+            None
+        )
+        current_app.logger.debug(f"Selected language level: {selected_language_level}")
+        language_level_context = (f"{selected_language_level['description']}, "
+                                  f"corresponding to CEFR level {selected_language_level['cefr_level']}")
+
+        flow_inputs = {
+            "region": arguments.region,
+            "working_schedule": arguments.working_schedule,
+            "start_date": arguments.start_date,
+            "language": arguments.language,
+            "interaction_mode": arguments.interaction_mode,
+            'tone_of_voice': tone_of_voice,
+            'tone_of_voice_context': tone_of_voice_context,
+            'language_level': language_level,
+            'language_level_context': language_level_context,
+            'ko_criteria': ko_competencies,
+        }
+
+        flow_results = self.flow.kickoff(inputs=flow_inputs)
+
+        current_app.logger.debug(f"Flow results: {flow_results}")
+
+        current_app.logger.debug(f"Flow state: {self.flow.state}")
+
+        fields = {}
+        for ko_question in self.flow.state.ko_criteria_questions:
+            fields[ko_question.title] = {
+                "name": ko_question.title,
+                "description": ko_question.title,
+                "context": ko_question.question,
+                "type": "options",
+                "required": True,
+                "allowed_values": [ko_question.answer_positive, ko_question.answer_negative]
+            }
+
+        ko_form = {
+            "type": "KO_CRITERIA_FORM",
+            "version": "1.0.0",
+            "name": "Starter Questions",
+            "icon": "verified",
+            "fields": fields,
+        }
+
+        results = SpecialistResult.create_for_type(self.type, self.type_version,
+                                                   answer=f"We starten met een aantal KO Criteria vragen",
+                                                   form_request=ko_form,
+                                                   phase="ko_question_evaluation")
+
+        return results
+
+    def execute_ko_question_evaluation(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("Traicie Selection Specialist ko_question_evaluation started", {})
+
+        # Check if the form has been returned (it should)
+        if not arguments.form_values:
+            raise EveAISpecialistExecutionError(self.tenant_id, self.specialist_id, self.session_id, "No form values returned")
+        current_app.logger.debug(f"Form values: {arguments.form_values}")
+
+        # Load the previous KO Questions
+        previous_ko_questions = self.flow.state.ko_criteria_questions
+        current_app.logger.debug(f"Previous KO Questions: {previous_ko_questions}")
+
+        # Evaluate KO Criteria
+        evaluation = "positive"
+        for criterium, answer in arguments.form_values.items():
+            for qa in previous_ko_questions:
+                if qa.get("title") == criterium:
+                    if qa.get("answer_positive") != answer:
+                        evaluation = "negative"
+                        break
+            if evaluation == "negative":
+                break
+
+        if evaluation == "negative":
+            results = SpecialistResult.create_for_type(self.type, self.type_version,
+                                                       answer=f"We hebben de antwoorden op de KO criteria verwerkt. Je voldoet jammer genoeg niet aan de minimale vereisten voor deze job.",
+                                                       form_request=None,
+                                                       phase="no_valid_candidate")
+        else:
+            # Check if answers to questions are positive
+            contact_form = cache_manager.specialist_forms_config_cache.get_config("PERSONAL_CONTACT_FORM", "1.0")
+            results = SpecialistResult.create_for_type(self.type, self.type_version,
+                                                       answer=f"We hebben de antwoorden op de KO criteria verwerkt. Je bent een geschikte kandidaat. Kan je je contactegevens doorgeven?",
+                                                       form_request=contact_form,
+                                                       phase="personal_contact_data")
+
+        return results
+
+    def execute_personal_contact_data(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("Traicie Selection Specialist personal_contact_data started", {})
+
+        results = SpecialistResult.create_for_type(self.type, self.type_version,
+                                                   answer=f"We hebben de contactgegevens verwerkt. We nemen zo snel mogelijk contact met je op.",
+                                                   phase="candidate_selected")
+        return results
+
+    def execute_no_valid_candidate(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("Traicie Selection Specialist no_valid_candidate started", {})
+        results = SpecialistResult.create_for_type(self.type, self.type_version,
+                                                   answer=f"Je voldoet jammer genoeg niet aan de minimale vereisten voor deze job. Maar solliciteer gerust voor één van onze andere jobs.",
+                                                   phase="no_valid_candidate")
+
+    def execute_candidate_selected(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("Traicie Selection Specialist candidate_selected started", {})
+        results = SpecialistResult.create_for_type(self.type, self.type_version,
+                                                   answer=f"We hebben je contactgegegevens verwerkt. We nemen zo snel mogelijk contact met je op.",
+                                                   phase="candidate_selected")
+        return results
+
+
+class SelectionInput(BaseModel):
+    region: str = Field(..., alias="region")
+    working_schedule: Optional[str] = Field(..., alias="working_schedule")
+    start_date: Optional[date] = Field(None, alias="vacancy_text")
+    language: Optional[str] = Field(None, alias="language")
+    interaction_mode: Optional[str] = Field(None, alias="interaction_mode")
+    tone_of_voice: Optional[str] = Field(None, alias="tone_of_voice")
+    tone_of_voice_context: Optional[str] = Field(None, alias="tone_of_voice_context")
+    language_level: Optional[str] = Field(None, alias="language_level")
+    language_level_context: Optional[str] = Field(None, alias="language_level_context")
+    ko_criteria: Optional[List[Dict[str, str]]] = Field(None, alias="ko_criteria")
+    question: Optional[str] = Field(None, alias="question")
+    field_values: Optional[Dict[str, Any]] = Field(None, alias="field_values")
+
+
+class SelectionKOCriteriumScore(BaseModel):
+    criterium: Optional[str] = Field(None, alias="criterium")
+    answer: Optional[str] = Field(None, alias="answer")
+    score: Optional[int] = Field(None, alias="score")
+
+
+class SelectionCompetencyScore(BaseModel):
+    competency: Optional[str] = Field(None, alias="competency")
+    answer: Optional[str] = Field(None, alias="answer")
+    score: Optional[int] = Field(None, alias="score")
+
+
+class PersonalContactData(BaseModel):
+    name: str = Field(..., description="Your name", alias="name")
+    email: EmailStr = Field(..., description="Your Name", alias="email")
+    phone: str = Field(..., description="Your Phone Number", alias="phone")
+    address: Optional[str] = Field(None, description="Your Address", alias="address")
+    zip: Optional[str] = Field(None, description="Postal Code", alias="zip")
+    city: Optional[str] = Field(None, description="City", alias="city")
+    country: Optional[str] = Field(None, description="Country", alias="country")
+    consent: bool = Field(..., description="Consent", alias="consent")
+
+
+class SelectionResult(SpecialistResult):
+    ko_criteria_questions: Optional[List[ListItem]] = Field(None, alias="ko_criteria_questions")
+    ko_criteria_scores: Optional[List[SelectionKOCriteriumScore]] = Field(None, alias="ko_criteria_scores")
+    competency_questions: Optional[List[ListItem]] = Field(None, alias="competency_questions")
+    competency_scores: Optional[List[SelectionCompetencyScore]] = Field(None, alias="competency_scores")
+    personal_contact_data: Optional[PersonalContactData] = Field(None, alias="personal_contact_data")
+
+
+class SelectionFlowState(EveAIFlowState):
+    """Flow state for Traicie Role Definition specialist that automatically updates from task outputs"""
+    input: Optional[SelectionInput] = None
+    ko_criteria_questions: Optional[List[KOQuestion]] = Field(None, alias="ko_criteria_questions")
+    ko_criteria_scores: Optional[List[SelectionKOCriteriumScore]] = Field(None, alias="ko_criteria_scores")
+    competency_questions: Optional[List[ListItem]] = Field(None, alias="competency_questions")
+    competency_scores: Optional[List[SelectionCompetencyScore]] = Field(None, alias="competency_scores")
+    personal_contact_data: Optional[PersonalContactData] = Field(None, alias="personal_contact_data")
+    phase: Optional[str] = Field(None, alias="phase")
+    interaction_mode: Optional[str] = Field(None, alias="mode")
+
+
+class SelectionFlow(EveAICrewAIFlow[SelectionFlowState]):
+    def __init__(self,
+                 specialist_executor: CrewAIBaseSpecialistExecutor,
+                 ko_def_crew: EveAICrewAICrew,
+                 **kwargs):
+        super().__init__(specialist_executor, "Traicie Role Definition Specialist Flow", **kwargs)
+        self.specialist_executor = specialist_executor
+        self.ko_def_crew = ko_def_crew
+        self.exception_raised = False
+
+    @start()
+    def process_inputs(self):
+        return ""
+
+    @listen(process_inputs)
+    async def execute_ko_def_definition(self):
+        inputs = self.state.input.model_dump()
+        try:
+            current_app.logger.debug("execute_ko_interview_definition")
+            crew_output = await self.ko_def_crew.kickoff_async(inputs=inputs)
+            # Unfortunately, crew_output will only contain the output of the latest task.
+            # As we will only take into account the flow state, we need to ensure both competencies and criteria
+            # are copies to the flow state.
+            update = {}
+            for task in self.ko_def_crew.tasks:
+                current_app.logger.debug(f"Task {task.name} output:\n{task.output}")
+                if task.name == "traicie_ko_criteria_interview_definition_task":
+                    # update["competencies"] = task.output.pydantic.competencies
+                    self.state.ko_criteria_questions = task.output.pydantic.ko_questions
+            # crew_output.pydantic = crew_output.pydantic.model_copy(update=update)
+            self.state.phase = "personal_contact_data"
+            current_app.logger.debug(f"State after execute_ko_def_definition: {self.state}")
+            current_app.logger.debug(f"State dump after execute_ko_def_definition: {self.state.model_dump()}")
+            return crew_output
+        except Exception as e:
+            current_app.logger.error(f"CREW execute_ko_def Kickoff Error: {str(e)}")
+            self.exception_raised = True
+            raise e
+
+    async def kickoff_async(self, inputs=None):
+        current_app.logger.debug(f"Async kickoff {self.name}")
+        current_app.logger.debug(f"Inputs: {inputs}")
+        self.state.input = SelectionInput.model_validate(inputs)
+        current_app.logger.debug(f"State: {self.state}")
+        result = await super().kickoff_async(inputs)
+        return self.state

eveai_entitlements/__init__.py

@@ -5,7 +5,7 @@ import os
 
 from common.utils.celery_utils import make_celery, init_celery
 from common.extensions import db, minio_client, cache_manager
-from config.logging_config import LOGGING
+from config.logging_config import configure_logging
 from config.config import get_config
 
 
@@ -22,7 +22,7 @@ def create_app(config_file=None):
         case _:
             app.config.from_object(get_config('dev'))
 
-    logging.config.dictConfig(LOGGING)
+    configure_logging()
 
     register_extensions(app)
 

eveai_workers/__init__.py

@@ -5,7 +5,7 @@ import os
 
 from common.utils.celery_utils import make_celery, init_celery
 from common.extensions import db, minio_client, cache_manager
-import config.logging_config as logging_config
+from config.logging_config import configure_logging
 from config.config import get_config
 
 
@@ -22,7 +22,7 @@ def create_app(config_file=None):
         case _:
             app.config.from_object(get_config('dev'))
 
-    logging.config.dictConfig(logging_config.LOGGING)
+    configure_logging()
 
     register_extensions(app)
 

eveai_workers/processors/__init__.py

@@ -1,5 +1,5 @@
 # Import all processor implementations to ensure registration
-from . import audio_processor, html_processor, pdf_processor, markdown_processor, docx_processor
+from . import audio_processor, html_processor, pdf_processor, markdown_processor, docx_processor, automagic_html_processor
 
 # List of all available processor implementations
-__all__ = ['audio_processor', 'html_processor', 'pdf_processor', 'markdown_processor', 'docx_processor']
+__all__ = ['audio_processor', 'html_processor', 'pdf_processor', 'markdown_processor', 'docx_processor', 'automagic_html_processor']

eveai_workers/processors/automagic_html_processor.py

@@ -0,0 +1,65 @@
+import io
+import pdfplumber
+from flask import current_app
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+import re
+from langchain_core.runnables import RunnablePassthrough
+
+from common.eveai_model.tracked_mistral_ocr_client import TrackedMistralOcrClient
+from common.extensions import minio_client
+from common.utils.model_utils import create_language_template, get_embedding_llm, get_template
+from .base_processor import BaseProcessor
+from common.utils.business_event_context import current_event
+from .processor_registry import ProcessorRegistry
+
+
+class AutomagicHTMLProcessor(BaseProcessor):
+    def __init__(self, tenant, document_version, catalog, processor):
+        super().__init__(tenant, document_version, catalog, processor)
+
+        self.chunk_size = catalog.max_chunk_size
+        self.chunk_overlap = 0
+        self.tuning = self.processor.tuning
+
+        self.prompt_params = {
+            "custom_instructions": self.processor.configuration.get("custom_instructions", ""),
+        }
+        template, llm = get_template("automagic_html_parse")
+
+        translation_prompt = ChatPromptTemplate.from_template(template)
+        setup = RunnablePassthrough()
+        output_parser = StrOutputParser()
+        self.chain = (setup | translation_prompt | llm | output_parser)
+
+
+    def process(self):
+        self._log("Starting Automagic HTML processing")
+        try:
+            # Get HTML-file data
+            file_data = minio_client.download_document_file(
+                self.tenant.id,
+                self.document_version.bucket_name,
+                self.document_version.object_name,
+            )
+
+            # Invoke HTML Processing Agent
+            self.prompt_params["html"] = file_data
+            with current_event.create_span("Markdown Generation"):
+                markdown = self.chain.invoke(self.prompt_params)
+            self._save_markdown(markdown)
+
+            # Retrieve Title
+            match = re.search(r'^# (.+)', markdown, re.MULTILINE)
+            title = match.group(1).strip() if match else None
+
+            self._log("Finished Automagic HTML Processing")
+            return markdown, title
+        except Exception as e:
+            self._log(f"Error automagically processing HTML: {str(e)}", level='error')
+            raise
+
+
+# Register the processor
+ProcessorRegistry.register("AUTOMAGIC_HTML_PROCESSOR", AutomagicHTMLProcessor)

eveai_workers/processors/pdf_processor.py

@@ -44,185 +44,6 @@ class PDFProcessor(BaseProcessor):
             self._log(f"Error processing PDF: {str(e)}", level='error')
             raise
 
-    def _extract_content(self, file_data):
-        extracted_content = []
-        with pdfplumber.open(io.BytesIO(file_data)) as pdf:
-            figure_counter = 1
-            for page_num, page in enumerate(pdf.pages):
-                self._log(f"Extracting content from page {page_num + 1}")
-                page_content = {
-                    'text': page.extract_text(),
-                    'figures': self._extract_figures(page, page_num, figure_counter),
-                    'tables': self._extract_tables(page)
-                }
-                self.log_tuning("_extract_content", {"page_num": page_num, "page_content": page_content})
-                figure_counter += len(page_content['figures'])
-                extracted_content.append(page_content)
-
-        return extracted_content
-
-    def _extract_figures(self, page, page_num, figure_counter):
-        figures = []
-        # Omit figure processing for now!
-        # for img in page.images:
-        #     try:
-        #         # Try to get the bbox, use full page dimensions if not available
-        #         bbox = img.get('bbox', (0, 0, page.width, page.height))
-        #
-        #         figure = {
-        #             'figure_number': figure_counter,
-        #             'filename': f"figure_{page_num + 1}_{figure_counter}.png",
-        #             'caption': self._find_figure_caption(page, bbox)
-        #         }
-        #
-        #         # Extract the figure as an image
-        #         figure_image = page.within_bbox(bbox).to_image()
-        #
-        #         # Save the figure using MinIO
-        #         with io.BytesIO() as output:
-        #             figure_image.save(output, format='PNG')
-        #             output.seek(0)
-        #             minio_client.upload_document_file(
-        #                 self.tenant.id,
-        #                 self.document_version.doc_id,
-        #                 self.document_version.language,
-        #                 self.document_version.id,
-        #                 figure['filename'],
-        #                 output.getvalue()
-        #             )
-        #
-        #         figures.append(figure)
-        #         figure_counter += 1
-        #     except Exception as e:
-        #         self._log(f"Error processing figure on page {page_num + 1}: {str(e)}", level='error')
-
-        return figures
-
-    def _find_figure_caption(self, page, bbox):
-        try:
-            # Look for text below the figure
-            caption_bbox = (bbox[0], bbox[3], bbox[2], min(bbox[3] + 50, page.height))
-            caption_text = page.crop(caption_bbox).extract_text()
-            if caption_text and caption_text.lower().startswith('figure'):
-                return caption_text
-        except Exception as e:
-            self._log(f"Error finding figure caption: {str(e)}", level='error')
-        return None
-
-    def _extract_tables(self, page):
-        tables = []
-        try:
-            for table in page.extract_tables():
-                if table:
-                    markdown_table = self._table_to_markdown(table)
-                    if markdown_table:  # Only add non-empty tables
-                        tables.append(markdown_table)
-                        self.log_tuning("_extract_tables", {"markdown_table": markdown_table})
-        except Exception as e:
-            self._log(f"Error extracting tables from page: {str(e)}", level='error')
-        return tables
-
-    def _table_to_markdown(self, table):
-        if not table or not table[0]:  # Check if table is empty or first row is empty
-            return ""  # Return empty string for empty tables
-
-        def clean_cell(cell):
-            if cell is None:
-                return ""  # Convert None to empty string
-            return str(cell).replace("|", "\\|")  # Escape pipe characters and convert to string
-
-        header = [clean_cell(cell) for cell in table[0]]
-        markdown = "| " + " | ".join(header) + " |\n"
-        markdown += "| " + " | ".join(["---"] * len(header)) + " |\n"
-
-        for row in table[1:]:
-            cleaned_row = [clean_cell(cell) for cell in row]
-            markdown += "| " + " | ".join(cleaned_row) + " |\n"
-
-        return markdown
-
-    def _structure_content(self, extracted_content):
-        structured_content = ""
-        title = "Untitled Document"
-        current_heading_level = 0
-        heading_pattern = re.compile(r'^(\d+(\.\d+)*\.?\s*)?(.+)$')
-
-        def identify_heading(text):
-            match = heading_pattern.match(text.strip())
-            if match:
-                numbering, _, content = match.groups()
-                if numbering:
-                    level = numbering.count('.') + 1
-                    return level, f"{numbering}{content}"
-                else:
-                    return 1, content  # Assume it's a top-level heading if no numbering
-            return 0, text  # Not a heading
-
-        for page in extracted_content:
-            # Assume the title is on the first page
-            if page == extracted_content[0]:
-                lines = page.get('text', '').split('\n')
-                if lines:
-                    title = lines[0].strip()  # Use the first non-empty line as the title
-
-            # Process text
-            paragraphs = page['text'].split('\n\n')
-
-            for para in paragraphs:
-                lines = para.strip().split('\n')
-                if len(lines) == 1:  # Potential heading
-                    level, text = identify_heading(lines[0])
-                    if level > 0:
-                        heading_marks = '#' * level
-                        structured_content += f"\n\n{heading_marks} {text}\n\n"
-                        if level == 1 and not title:
-                            title = text  # Use the first top-level heading as the title if not set
-                    else:
-                        structured_content += f"{para}\n\n"  # Treat as normal paragraph
-                else:
-                    structured_content += f"{para}\n\n"  # Multi-line paragraph
-
-            # Process figures
-            for figure in page.get('figures', []):
-                structured_content += f"\n\n![Figure {figure['figure_number']}]({figure['filename']})\n\n"
-                if figure['caption']:
-                    structured_content += f"*Figure {figure['figure_number']}: {figure['caption']}*\n\n"
-
-            # Add tables
-            if 'tables' in page:
-                for table in page['tables']:
-                    structured_content += f"\n{table}\n"
-
-        if self.tuning:
-            self._save_intermediate(structured_content, "structured_content.md")
-
-        return structured_content, title
-
-    def _split_content_for_llm(self, content):
-        text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=self.chunk_size,
-            chunk_overlap=self.chunk_overlap,
-            length_function=len,
-            separators=["\n\n", "\n", " ", ""]
-        )
-        return text_splitter.split_text(content)
-
-    def _process_chunks_with_llm(self, chunks):
-        template, llm = get_template('pdf_parse')
-        pdf_prompt = ChatPromptTemplate.from_template(template)
-        setup = RunnablePassthrough()
-        output_parser = StrOutputParser()
-        chain = setup | pdf_prompt | llm | output_parser
-
-        markdown_chunks = []
-        for chunk in chunks:
-            input = {"pdf_content": chunk}
-            result = chain.invoke(input)
-            result = self._clean_markdown(result)
-            markdown_chunks.append(result)
-
-        return "\n\n".join(markdown_chunks)
-
 
 # Register the processor
 ProcessorRegistry.register("PDF_PROCESSOR", PDFProcessor)

eveai_workers/tasks.py

@@ -11,6 +11,7 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.runnables import RunnablePassthrough
 from sqlalchemy import or_
 from sqlalchemy.exc import SQLAlchemyError
+import traceback
 
 from common.extensions import db, cache_manager
 from common.models.document import DocumentVersion, Embedding, Document, Processor, Catalog
@@ -24,7 +25,8 @@ from common.utils.business_event_context import current_event
 from config.type_defs.processor_types import PROCESSOR_TYPES
 from eveai_workers.processors.processor_registry import ProcessorRegistry
 
-from common.utils.eveai_exceptions import EveAIInvalidEmbeddingModel
+from common.utils.eveai_exceptions import EveAIInvalidEmbeddingModel, EveAINoContentFound, EveAIUnsupportedFileType, \
+    EveAINoProcessorFound
 
 from common.utils.config_field_types import json_to_pattern_list
 
@@ -58,8 +60,8 @@ def create_embeddings(tenant_id, document_version_id):
         catalog = Catalog.query.get_or_404(catalog_id)
 
         # Define processor related information
-        processor_type, processor_class = ProcessorRegistry.get_processor_for_file_type(document_version.file_type)
         processor = get_processor_for_document(catalog_id, document_version.file_type, document_version.sub_file_type)
+        processor_class = ProcessorRegistry.get_processor_class(processor.type)
 
     except Exception as e:
         current_app.logger.error(f'Create Embeddings request received '
@@ -95,7 +97,7 @@ def create_embeddings(tenant_id, document_version_id):
         delete_embeddings_for_document_version(document_version)
 
         try:
-            with current_event.create_span(f"{processor_type} Processing"):
+            with current_event.create_span(f"{processor.type} Processing"):
                 document_processor = processor_class(
                     tenant=tenant,
                     document_version=document_version,
@@ -107,6 +109,8 @@ def create_embeddings(tenant_id, document_version_id):
                     'markdown': markdown,
                     'title': title
                 })
+                if not markdown or markdown.strip() == '':
+                    raise EveAINoContentFound(document_version.doc_id, document_version.id)
 
             with current_event.create_span("Embedding"):
                 embed_markdown(tenant, document_version, catalog, document_processor, markdown, title)
@@ -114,9 +118,11 @@ def create_embeddings(tenant_id, document_version_id):
             current_event.log("Finished Embedding Creation Task")
 
         except Exception as e:
+            stacktrace = traceback.format_exc()
             current_app.logger.error(f'Error creating embeddings for tenant {tenant_id} '
                                  f'on document version {document_version_id} '
-                                     f'error: {e}')
+                                 f'error: {e}\n'
+                                 f'Stacktrace: {stacktrace}')
             document_version.processing = False
             document_version.processing_finished_at = dt.now(tz.utc)
             document_version.processing_error = str(e)[:255]
@@ -624,25 +630,9 @@ def get_processor_for_document(catalog_id: int, file_type: str, sub_file_type: s
         ValueError: If no matching processor is found
     """
     try:
+        current_app.logger.debug(f"Getting processor for catalog {catalog_id}, file type {file_type}, file sub_type {sub_file_type} ")
         # Start with base query for catalog
-        query = Processor.query.filter_by(catalog_id=catalog_id)
+        query = Processor.query.filter_by(catalog_id=catalog_id).filter_by(active=True)
-
-        # Find processor type that handles this file type
-        matching_processor_type = None
-        for proc_type, config in PROCESSOR_TYPES.items():
-            supported_types = config['file_types']
-            if isinstance(supported_types, str):
-                supported_types = [t.strip() for t in supported_types.split(',')]
-
-            if file_type in supported_types:
-                matching_processor_type = proc_type
-                break
-
-        if not matching_processor_type:
-            raise ValueError(f"No processor type found for file type: {file_type}")
-
-        # Add processor type condition
-        query = query.filter_by(type=matching_processor_type)
 
         # If sub_file_type is provided, add that condition
         if sub_file_type:
@@ -652,21 +642,43 @@ def get_processor_for_document(catalog_id: int, file_type: str, sub_file_type: s
             query = query.filter(or_(Processor.sub_file_type.is_(None),
                                      Processor.sub_file_type == ''))
         
-        # Get the first matching processor
+        available_processors = query.all()
-        processor = query.first()
+
+        if not available_processors:
+            raise EveAINoProcessorFound(catalog_id, file_type, sub_file_type)
+        available_processor_types = [processor.type for processor in available_processors]
+        current_app.logger.debug(f"Available processors for catalog {catalog_id}: {available_processor_types}")
+
+        # Find processor type that handles this file type
+        matching_processor_type = None
+        for proc_type, config in PROCESSOR_TYPES.items():
+            # Alleen verwerken als dit type processor beschikbaar is in de database
+            if proc_type in available_processor_types:
+                supported_types = config['file_types']
+                if isinstance(supported_types, str):
+                    supported_types = [t.strip() for t in supported_types.split(',')]
+                    current_app.logger.debug(f"Supported types for processor type {proc_type}: {supported_types}")
+
+                if file_type in supported_types:
+                    matching_processor_type = proc_type
+                    break
+
+        current_app.logger.debug(f"Processor type found for catalog {catalog_id}, file type {file_type}: {matching_processor_type}")
+        if not matching_processor_type:
+            raise EveAINoProcessorFound(catalog_id, file_type, sub_file_type)
+        else:
+            current_app.logger.debug(f"Processor type found for file type: {file_type}: {matching_processor_type}")
+
+        processor = None
+        for proc in available_processors:
+            if proc.type == matching_processor_type:
+                processor = proc
+                break
 
         if not processor:
-            if sub_file_type:
+            raise EveAINoProcessorFound(catalog_id, file_type, sub_file_type)
-                raise ValueError(
-                    f"No processor found for catalog {catalog_id} of type {matching_processor_type}, "
-                    f"file type {file_type}, sub-type {sub_file_type}"
-                )
-            else:
-                raise ValueError(
-                    f"No processor found for catalog {catalog_id}, "
-                    f"file type {file_type}"
-                )
 
+        current_app.logger.debug(f"Processor found for catalog {catalog_id}, file type {file_type}: {processor}")
         return processor
 
     except Exception as e:

logs/.gitkeep

@@ -0,0 +1,2 @@
+# Deze directory bevat logbestanden
+# .gitkeep zorgt ervoor dat de directory wordt meegenomen in Git

migrations/public/versions/e47dc002b678_add_allowed_languages_to_tenantmake_.py

@@ -0,0 +1,53 @@
+"""Add allowed_languages to TenantMake, introduce TranslationCache
+
+Revision ID: e47dc002b678
+Revises: 83d4e90f87c6
+Create Date: 2025-06-26 13:43:43.719865
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e47dc002b678'
+down_revision = '83d4e90f87c6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('translation_cache',
+    sa.Column('cache_key', sa.String(length=16), nullable=False),
+    sa.Column('source_text', sa.Text(), nullable=False),
+    sa.Column('translated_text', sa.Text(), nullable=False),
+    sa.Column('source_language', sa.String(length=2), nullable=False),
+    sa.Column('target_language', sa.String(length=2), nullable=False),
+    sa.Column('context', sa.Text(), nullable=True),
+    sa.Column('prompt_tokens', sa.Integer(), nullable=False),
+    sa.Column('completion_tokens', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+    sa.Column('created_by', sa.Integer(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_by', sa.Integer(), nullable=True),
+    sa.Column('last_used_at', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['created_by'], ['public.user.id'], ),
+    sa.ForeignKeyConstraint(['updated_by'], ['public.user.id'], ),
+    sa.PrimaryKeyConstraint('cache_key'),
+    schema='public'
+    )
+
+    with op.batch_alter_table('tenant_make', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('allowed_languages', postgresql.ARRAY(sa.String(length=2)), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tenant_make', schema=None) as batch_op:
+        batch_op.drop_column('allowed_languages')
+
+        op.drop_table('translation_cache', schema='public')
+    # ### end Alembic commands ###

migrations/tenant/env.py

@@ -72,7 +72,8 @@ def get_public_table_names():
     # TODO: This function should include the necessary functionality to automatically retrieve table names
     return ['role', 'roles_users', 'tenant', 'user', 'tenant_domain','license_tier', 'license', 'license_usage',
             'business_event_log', 'tenant_project', 'partner', 'partner_service', 'invoice', 'license_period',
-            'license_change_log', 'partner_service_license_tier', 'payment', 'partner_tenant']
+            'license_change_log', 'partner_service_license_tier', 'payment', 'partner_tenant', 'tenant_make',
+            'specialist_magic_link_tenant']
 
 PUBLIC_TABLES = get_public_table_names()
 logger.info(f"Public tables: {PUBLIC_TABLES}")

migrations/tenant/versions/b1647f31339a_add_active_flag_to_processor.py

@@ -0,0 +1,30 @@
+"""Add Active Flag to Processor
+
+Revision ID: b1647f31339a
+Revises: 2b6ae6cc923e
+Create Date: 2025-06-25 12:34:35.391516
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'b1647f31339a'
+down_revision = '2b6ae6cc923e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('processor', sa.Column('active', sa.Boolean(), nullable=True))
+    op.execute("UPDATE processor SET active = true")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('processor', 'active')
+    # ### end Alembic commands ###

requirements-k8s.txt

@@ -0,0 +1,2 @@
+# Extra vereisten voor Kubernetes-omgeving
+python-json-logger>=2.0.7

requirements.txt

@@ -93,3 +93,6 @@ prometheus_client~=0.21.1
 scaleway~=2.9.0
 html2text~=2025.4.15
 markdown~=3.8
+python-json-logger~=2.0.7
+qrcode[pil]==8.2
+xxhash~=3.5.0

scripts/check_logs.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+
+"""
+Dit script controleert of de logs directory bestaat en toegankelijk is,
+en test of logging correct werkt.
+"""
+
+import os
+import sys
+import logging
+import traceback
+
+def check_logs_directory():
+    # Verkrijg het absolute pad naar de logs directory
+    base_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+    logs_dir = os.path.join(base_dir, 'logs')
+
+    print(f"\nControleren van logs directory: {logs_dir}")
+
+    # Controleer of de directory bestaat
+    if not os.path.exists(logs_dir):
+        print("  - Directory bestaat niet. Proberen aan te maken...")
+        try:
+            os.makedirs(logs_dir, exist_ok=True)
+            print("  - Directory succesvol aangemaakt.")
+        except Exception as e:
+            print(f"  - FOUT: Kan directory niet aanmaken: {e}")
+            return False
+    else:
+        print("  - Directory bestaat.")
+
+    # Controleer schrijfrechten
+    if not os.access(logs_dir, os.W_OK):
+        print("  - FOUT: Geen schrijfrechten voor de logs directory.")
+        return False
+    else:
+        print("  - Directory is schrijfbaar.")
+
+    # Probeer een testbestand te schrijven
+    test_file = os.path.join(logs_dir, 'test_write.log')
+    try:
+        with open(test_file, 'w') as f:
+            f.write('Test schrijven naar logs directory.\n')
+        print(f"  - Succesvol testbestand geschreven naar {test_file}")
+        os.remove(test_file)  # Verwijder het testbestand
+        print("  - Testbestand verwijderd.")
+    except Exception as e:
+        print(f"  - FOUT: Kan niet schrijven naar logs directory: {e}")
+        return False
+
+    return True
+
+def check_logging_config():
+    print("\nControleren van logging configuratie...")
+
+    try:
+        from config.logging_config import configure_logging
+        configure_logging()
+        print("  - Logging configuratie geladen.")
+
+        # Test enkele loggers
+        loggers_to_test = ['eveai_app', 'eveai_workers', 'eveai_api', 'tuning']
+        for logger_name in loggers_to_test:
+            logger = logging.getLogger(logger_name)
+            logger.info(f"Test log bericht van {logger_name}")
+            print(f"  - Logger '{logger_name}' getest.")
+
+        print("  - Alle loggers succesvol getest.")
+        return True
+
+    except Exception as e:
+        print(f"  - FOUT bij laden van logging configuratie: {e}")
+        traceback.print_exc()
+        return False
+
+def main():
+    print("\nEveAI Logging Test Utility")
+    print("===========================\n")
+
+    directory_ok = check_logs_directory()
+    if not directory_ok:
+        print("\nPROBLEEM: De logs directory is niet toegankelijk of schrijfbaar.")
+        print("Oplossingen:")
+        print("  1. Zorg ervoor dat de gebruiker die de applicatie uitvoert schrijfrechten heeft voor de logs directory.")
+        print("  2. Voer het commando uit: mkdir -p logs && chmod 777 logs")
+
+    config_ok = check_logging_config()
+    if not config_ok:
+        print("\nPROBLEEM: De logging configuratie kon niet worden geladen.")
+        print("Controleer de config/logging_config.py file.")
+
+    if directory_ok and config_ok:
+        print("\nALLES OK: Logging lijkt correct geconfigureerd.")
+        print("Controleer de logbestanden in de 'logs' directory voor de testberichten.")
+    else:
+        print("\nEr zijn problemen gevonden die opgelost moeten worden.")
+        return 1
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())