- Release script added to tag in both git and docker

- Small bug fixes

.gitignore

@@ -41,3 +41,5 @@ migrations/.DS_Store
 migrations/public/.DS_Store
 scripts/.DS_Store
 scripts/__pycache__/run_eveai_app.cpython-312.pyc
+/eveai_repo.txt
+*repo.txt

.idea/sqldialects.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="SqlDialectMappings">
-    <file url="PROJECT" dialect="PostgreSQL" />
-  </component>
-</project>

.repopackignore_base

@@ -0,0 +1,20 @@
+# Add patterns to ignore here, one per line
+# Example:
+# *.log
+# tmp/
+logs/
+nginx/static/assets/fonts/
+nginx/static/assets/img/
+nginx/static/assets/js/
+nginx/static/scss/
+patched_packages/
+migrations/
+*material*
+*nucleo*
+*package*
+nginx/mime.types
+*.gitignore*
+.python-version
+.repopackignore*
+repopack.config.json
+*repo.txt

.repopackignore_components

@@ -0,0 +1,12 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_docker

@@ -0,0 +1,12 @@
+common/
+config/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/

.repopackignore_eveai_api

@@ -0,0 +1,11 @@
+docker/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_eveai_app

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_eveai_beat

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_eveai_chat

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_eveai_chat_workers

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_eveai_entitlements

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_workers/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_eveai_workers

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+instance/
+integrations/
+nginx/
+scripts/

.repopackignore_full

@@ -0,0 +1,4 @@
+docker
+integrations
+nginx
+scripts

.repopackignore_integrations

@@ -0,0 +1,13 @@
+common/
+config/
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+nginx/
+scripts/

.repopackignore_nginx

@@ -0,0 +1,11 @@
+docker/
+eveai_api/
+eveai_app/
+eveai_beat/
+eveai_chat/
+eveai_chat_workers/
+eveai_entitlements/
+eveai_workers/
+instance/
+integrations/
+scripts/

CHANGELOG.md

@@ -24,7 +24,110 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 - In case of vulnerabilities.
-- 
+
+## [1.0.13-alfa]
+
+### Added
+- Finished Catalog introduction
+- Reinitialization of WordPress site for syncing
+
+### Changed
+- Modification of WordPress Sync Component
+- Cleanup of attributes in Tenant
+
+### Fixed
+- Overall bugfixes as result from the Catalog introduction
+
+## [1.0.12-alfa]
+
+### Added
+- Added Catalog functionality
+
+### Changed
+- For changes in existing functionality.
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- Set default language when registering Documents or URLs.
+
+### Security
+- In case of vulnerabilities.
+
+## [1.0.11-alfa]
+
+### Added
+- License Usage Calculation realised
+- View License Usages
+- Celery Beat container added
+- First schedule in Celery Beat for calculating usage (hourly)
+
+### Changed
+- repopack can now split for different components
+
+### Fixed
+- Various fixes as consequence of changing file_location / file_name ==> bucket_name / object_name
+- Celery Routing / Queuing updated
+
+## [1.0.10-alfa]
+
+### Added
+- BusinessEventLog monitoring using Langchain native code
+
+### Changed
+- Allow longer audio files (or video) to be uploaded and processed
+- Storage and Embedding usage now expressed in MiB iso tokens (more logical)
+- Views for License / LicenseTier
+
+### Removed
+- Portkey removed for monitoring usage
+
+## [1.0.9-alfa] - 2024/10/01
+
+### Added
+- Business Event tracing (eveai_workers & eveai_chat_workers)
+- Flower Container added for monitoring
+
+### Changed
+- Healthcheck improvements
+- model_utils turned into a class with lazy loading
+
+### Deprecated
+- For soon-to-be removed features.
+
+### Removed
+- For now removed features.
+
+### Fixed
+- Set default language when registering Documents or URLs.
+
+## [1.0.8-alfa] - 2024-09-12
+
+### Added
+- Tenant type defined to allow for active, inactive, demo ... tenants
+- Search and filtering functionality on Tenants
+- Implementation of health checks (1st version)
+- Provision for Prometheus monitoring (no implementation yet)
+- Refine audio_processor and srt_processor to reduce duplicate code and support larger files
+- Introduction of repopack to reason in LLMs about the code
+
+### Fixed
+- Refine audio_processor and srt_processor to reduce duplicate code and support larger files
+
+## [1.0.7-alfa] - 2024-09-12
+
+### Added
+- Full Document API allowing for creation, updating and invalidation of documents.
+- Metadata fields (JSON) added to DocumentVersion, allowing end-users to add structured information
+- Wordpress plugin eveai_sync to synchronize Wordpress content with EveAI
+
+### Fixed
+- Maximal deduplication of code between views and api in document_utils.py
+
 ## [1.0.6-alfa] - 2024-09-03
 
 ### Fixed

common/extensions.py

@@ -10,8 +10,8 @@ from flask_jwt_extended import JWTManager
 from flask_session import Session
 from flask_wtf import CSRFProtect
 from flask_restx import Api
+from prometheus_flask_exporter import PrometheusMetrics
 
-from .utils.nginx_utils import prefixed_url_for
 from .utils.simple_encryption import SimpleEncryption
 from .utils.minio_utils import MinioClient
 
@@ -31,3 +31,4 @@ session = Session()
 api_rest = Api()
 simple_encryption = SimpleEncryption()
 minio_client = MinioClient()
+metrics = PrometheusMetrics.for_app_factory()

common/langchain/eveai_history_retriever.py

@@ -1,23 +1,31 @@
 from langchain_core.retrievers import BaseRetriever
 from sqlalchemy import asc
 from sqlalchemy.exc import SQLAlchemyError
-from pydantic import BaseModel, Field
+from pydantic import Field, BaseModel, PrivateAttr
 from typing import Any, Dict
 from flask import current_app
 
 from common.extensions import db
 from common.models.interaction import ChatSession, Interaction
-from common.utils.datetime_utils import get_date_in_timezone
+from common.utils.model_utils import ModelVariables
 
 
-class EveAIHistoryRetriever(BaseRetriever):
+class EveAIHistoryRetriever(BaseRetriever, BaseModel):
-    model_variables: Dict[str, Any] = Field(...)
+    _model_variables: ModelVariables = PrivateAttr()
-    session_id: str = Field(...)
+    _session_id: str = PrivateAttr()
 
-    def __init__(self, model_variables: Dict[str, Any], session_id: str):
+    def __init__(self, model_variables: ModelVariables, session_id: str):
         super().__init__()
-        self.model_variables = model_variables
+        self._model_variables = model_variables
-        self.session_id = session_id
+        self._session_id = session_id
+
+    @property
+    def model_variables(self) -> ModelVariables:
+        return self._model_variables
+
+    @property
+    def session_id(self) -> str:
+        return self._session_id
 
     def _get_relevant_documents(self, query: str):
         current_app.logger.debug(f'Retrieving history of interactions for query: {query}')

common/langchain/eveai_retriever.py

@@ -1,35 +1,44 @@
 from langchain_core.retrievers import BaseRetriever
 from sqlalchemy import func, and_, or_, desc
 from sqlalchemy.exc import SQLAlchemyError
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, PrivateAttr
 from typing import Any, Dict
 from flask import current_app
 
 from common.extensions import db
 from common.models.document import Document, DocumentVersion
 from common.utils.datetime_utils import get_date_in_timezone
+from common.utils.model_utils import ModelVariables
 
 
-class EveAIRetriever(BaseRetriever):
+class EveAIRetriever(BaseRetriever, BaseModel):
-    model_variables: Dict[str, Any] = Field(...)
+    _model_variables: ModelVariables = PrivateAttr()
-    tenant_info: Dict[str, Any] = Field(...)
+    _tenant_info: Dict[str, Any] = PrivateAttr()
 
-    def __init__(self, model_variables: Dict[str, Any], tenant_info: Dict[str, Any]):
+    def __init__(self, model_variables: ModelVariables, tenant_info: Dict[str, Any]):
         super().__init__()
-        self.model_variables = model_variables
+        current_app.logger.debug(f'Model variables type: {type(model_variables)}')
-        self.tenant_info = tenant_info
+        self._model_variables = model_variables
+        self._tenant_info = tenant_info
+
+    @property
+    def model_variables(self) -> ModelVariables:
+        return self._model_variables
+
+    @property
+    def tenant_info(self) -> Dict[str, Any]:
+        return self._tenant_info
 
     def _get_relevant_documents(self, query: str):
-
-
-
         current_app.logger.debug(f'Retrieving relevant documents for query: {query}')
         query_embedding = self._get_query_embedding(query)
+        current_app.logger.debug(f'Model Variables Private: {type(self._model_variables)}')
+        current_app.logger.debug(f'Model Variables Property: {type(self.model_variables)}')
         db_class = self.model_variables['embedding_db_model']
         similarity_threshold = self.model_variables['similarity_threshold']
         k = self.model_variables['k']
 
-        if self.tenant_info['rag_tuning']:
+        if self.model_variables['rag_tuning']:
             try:
                 current_date = get_date_in_timezone(self.tenant_info['timezone'])
                 current_app.rag_tuning_logger.debug(f'Current date: {current_date}\n')
@@ -64,7 +73,7 @@ class EveAIRetriever(BaseRetriever):
                 current_app.logger.error(f'Error generating overview: {e}')
                 db.session.rollback()
 
-        if self.tenant_info['rag_tuning']:
+        if self.model_variables['rag_tuning']:
             current_app.rag_tuning_logger.debug(f'Parameters for Retrieval of documents: \n')
             current_app.rag_tuning_logger.debug(f'Similarity Threshold: {similarity_threshold}\n')
             current_app.rag_tuning_logger.debug(f'K: {k}\n')
@@ -97,14 +106,14 @@ class EveAIRetriever(BaseRetriever):
                 .limit(k)
             )
 
-            if self.tenant_info['rag_tuning']:
+            if self.model_variables['rag_tuning']:
                 current_app.rag_tuning_logger.debug(f'Query executed for Retrieval of documents: \n')
                 current_app.rag_tuning_logger.debug(f'{query_obj.statement}\n')
                 current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
 
             res = query_obj.all()
 
-            if self.tenant_info['rag_tuning']:
+            if self.model_variables['rag_tuning']:
                 current_app.rag_tuning_logger.debug(f'Retrieved {len(res)} relevant documents \n')
                 current_app.rag_tuning_logger.debug(f'Data retrieved: \n')
                 current_app.rag_tuning_logger.debug(f'{res}\n')
@@ -112,7 +121,7 @@ class EveAIRetriever(BaseRetriever):
 
             result = []
             for doc in res:
-                if self.tenant_info['rag_tuning']:
+                if self.model_variables['rag_tuning']:
                     current_app.rag_tuning_logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n')
                     current_app.rag_tuning_logger.debug(f'Chunk: \n {doc[0].chunk}\n\n')
                 result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n')

common/langchain/llm_metrics_handler.py

@@ -0,0 +1,49 @@
+import time
+from langchain.callbacks.base import BaseCallbackHandler
+from typing import Dict, Any, List
+from langchain.schema import LLMResult
+from common.utils.business_event_context import current_event
+from flask import current_app
+
+
+class LLMMetricsHandler(BaseCallbackHandler):
+    def __init__(self):
+        self.total_tokens: int = 0
+        self.prompt_tokens: int = 0
+        self.completion_tokens: int = 0
+        self.start_time: float = 0
+        self.end_time: float = 0
+        self.total_time: float = 0
+
+    def reset(self):
+        self.total_tokens = 0
+        self.prompt_tokens = 0
+        self.completion_tokens = 0
+        self.start_time = 0
+        self.end_time = 0
+        self.total_time = 0
+
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        self.start_time = time.time()
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        self.end_time = time.time()
+        self.total_time = self.end_time - self.start_time
+
+        usage = response.llm_output.get('token_usage', {})
+        self.prompt_tokens += usage.get('prompt_tokens', 0)
+        self.completion_tokens += usage.get('completion_tokens', 0)
+        self.total_tokens = self.prompt_tokens + self.completion_tokens
+
+        metrics = self.get_metrics()
+        current_event.log_llm_metrics(metrics)
+        self.reset()  # Reset for the next call
+
+    def get_metrics(self) -> Dict[str, int | float]:
+        return {
+            'total_tokens': self.total_tokens,
+            'prompt_tokens': self.prompt_tokens,
+            'completion_tokens': self.completion_tokens,
+            'time_elapsed': self.total_time,
+            'interaction_type': 'LLM',
+        }

common/langchain/tracked_openai_embeddings.py

@@ -0,0 +1,51 @@
+from langchain_openai import OpenAIEmbeddings
+from typing import List, Any
+import time
+from common.utils.business_event_context import current_event
+
+
+class TrackedOpenAIEmbeddings(OpenAIEmbeddings):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        start_time = time.time()
+        result = super().embed_documents(texts)
+        end_time = time.time()
+
+        # Estimate token usage (OpenAI uses tiktoken for this)
+        import tiktoken
+        enc = tiktoken.encoding_for_model(self.model)
+        total_tokens = sum(len(enc.encode(text)) for text in texts)
+
+        metrics = {
+            'total_tokens': total_tokens,
+            'prompt_tokens': total_tokens,  # For embeddings, all tokens are prompt tokens
+            'completion_tokens': 0,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'Embedding',
+            }
+        current_event.log_llm_metrics(metrics)
+
+        return result
+
+    def embed_query(self, text: str) -> List[float]:
+        start_time = time.time()
+        result = super().embed_query(text)
+        end_time = time.time()
+
+        # Estimate token usage
+        import tiktoken
+        enc = tiktoken.encoding_for_model(self.model)
+        total_tokens = len(enc.encode(text))
+
+        metrics = {
+            'total_tokens': total_tokens,
+            'prompt_tokens': total_tokens,
+            'completion_tokens': 0,
+            'time_elapsed': end_time - start_time,
+            'interaction_type': 'Embedding',
+        }
+        current_event.log_llm_metrics(metrics)
+
+        return result

common/langchain/tracked_transcribe.py

@@ -0,0 +1,27 @@
+import time
+from common.utils.business_event_context import current_event
+
+
+def tracked_transcribe(client, *args, **kwargs):
+    start_time = time.time()
+
+    # Extract the file and model from kwargs if present, otherwise use defaults
+    file = kwargs.get('file')
+    model = kwargs.get('model', 'whisper-1')
+    duration = kwargs.pop('duration', 600)
+
+    result = client.audio.transcriptions.create(*args, **kwargs)
+    end_time = time.time()
+
+    # Token usage for transcriptions is actually the duration in seconds we pass, as the whisper model is priced per second transcribed
+
+    metrics = {
+        'total_tokens': duration,
+        'prompt_tokens': 0,  # For transcriptions, all tokens are considered "completion"
+        'completion_tokens': duration,
+        'time_elapsed': end_time - start_time,
+        'interaction_type': 'ASR',
+    }
+    current_event.log_llm_metrics(metrics)
+
+    return result

common/models/document.py

@@ -2,12 +2,49 @@ from common.extensions import db
 from .user import User, Tenant
 from pgvector.sqlalchemy import Vector
 from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import ARRAY
+import sqlalchemy as sa
+
+
+class Catalog(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+
+    # Embedding variables
+    html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
+    html_end_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'li'])
+    html_included_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
+    html_excluded_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
+    html_excluded_classes = db.Column(ARRAY(sa.String(200)), nullable=True)
+
+    min_chunk_size = db.Column(db.Integer, nullable=True, default=2000)
+    max_chunk_size = db.Column(db.Integer, nullable=True, default=3000)
+
+    # Embedding search variables ==> move to specialist?
+    es_k = db.Column(db.Integer, nullable=True, default=8)
+    es_similarity_threshold = db.Column(db.Float, nullable=True, default=0.4)
+
+    # Chat variables ==> Move to Specialist?
+    chat_RAG_temperature = db.Column(db.Float, nullable=True, default=0.3)
+    chat_no_RAG_temperature = db.Column(db.Float, nullable=True, default=0.5)
+
+    # Tuning enablers
+    embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
+    rag_tuning = db.Column(db.Boolean, nullable=True, default=False)    # Move to Specialist?
+
+    # Versioning Information
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
+    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
+    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
 
 class Document(db.Model):
     id = db.Column(db.Integer, primary_key=True)
+    # tenant_id = db.Column(db.Integer, db.ForeignKey(Tenant.id), nullable=False)
+    catalog_id = db.Column(db.Integer, db.ForeignKey(Catalog.id), nullable=True)
     name = db.Column(db.String(100), nullable=False)
-    tenant_id = db.Column(db.Integer, db.ForeignKey(Tenant.id), nullable=False)
     valid_from = db.Column(db.DateTime, nullable=True)
     valid_to = db.Column(db.DateTime, nullable=True)
 
@@ -28,9 +65,10 @@ class DocumentVersion(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     doc_id = db.Column(db.Integer, db.ForeignKey(Document.id), nullable=False)
     url = db.Column(db.String(200), nullable=True)
-    file_location = db.Column(db.String(255), nullable=True)
+    bucket_name = db.Column(db.String(255), nullable=True)
-    file_name = db.Column(db.String(200), nullable=True)
+    object_name = db.Column(db.String(200), nullable=True)
     file_type = db.Column(db.String(20), nullable=True)
+    file_size = db.Column(db.Float, nullable=True)
     language = db.Column(db.String(2), nullable=False)
     user_context = db.Column(db.Text, nullable=True)
     system_context = db.Column(db.Text, nullable=True)
@@ -55,12 +93,6 @@ class DocumentVersion(db.Model):
     def __repr__(self):
         return f"<DocumentVersion {self.document_language.document_id}.{self.document_language.language}>.{self.id}>"
 
-    def calc_file_location(self):
-        return f"{self.document.tenant_id}/{self.document.id}/{self.language}"
-
-    def calc_file_name(self):
-        return f"{self.id}.{self.file_type}"
-
 
 class Embedding(db.Model):
     __tablename__ = 'embeddings'

common/models/entitlements.py

@@ -0,0 +1,110 @@
+from common.extensions import db
+
+
+class BusinessEventLog(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    timestamp = db.Column(db.DateTime, nullable=False)
+    event_type = db.Column(db.String(50), nullable=False)
+    tenant_id = db.Column(db.Integer, nullable=False)
+    trace_id = db.Column(db.String(50), nullable=False)
+    span_id = db.Column(db.String(50))
+    span_name = db.Column(db.String(50))
+    parent_span_id = db.Column(db.String(50))
+    document_version_id = db.Column(db.Integer)
+    document_version_file_size = db.Column(db.Float)
+    chat_session_id = db.Column(db.String(50))
+    interaction_id = db.Column(db.Integer)
+    environment = db.Column(db.String(20))
+    llm_metrics_total_tokens = db.Column(db.Integer)
+    llm_metrics_prompt_tokens = db.Column(db.Integer)
+    llm_metrics_completion_tokens = db.Column(db.Integer)
+    llm_metrics_total_time = db.Column(db.Float)
+    llm_metrics_call_count = db.Column(db.Integer)
+    llm_interaction_type = db.Column(db.String(20))
+    message = db.Column(db.Text)
+    license_usage_id = db.Column(db.Integer, db.ForeignKey('public.license_usage.id'), nullable=True)
+    license_usage = db.relationship('LicenseUsage', backref='events')
+
+
+class License(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    tier_id = db.Column(db.Integer, db.ForeignKey('public.license_tier.id'),nullable=False)  # 'small', 'medium', 'custom'
+    start_date = db.Column(db.Date, nullable=False)
+    end_date = db.Column(db.Date, nullable=True)
+    currency = db.Column(db.String(20), nullable=False)
+    yearly_payment = db.Column(db.Boolean, nullable=False, default=False)
+    basic_fee = db.Column(db.Float, nullable=False)
+    max_storage_mb = db.Column(db.Integer, nullable=False)
+    additional_storage_price = db.Column(db.Float, nullable=False)
+    additional_storage_bucket = db.Column(db.Integer, nullable=False)
+    included_embedding_mb = db.Column(db.Integer, nullable=False)
+    additional_embedding_price = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_embedding_bucket = db.Column(db.Integer, nullable=False)
+    included_interaction_tokens = db.Column(db.Integer, nullable=False)
+    additional_interaction_token_price = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_interaction_bucket = db.Column(db.Integer, nullable=False)
+    overage_embedding = db.Column(db.Float, nullable=False, default=0)
+    overage_interaction = db.Column(db.Float, nullable=False, default=0)
+
+    tenant = db.relationship('Tenant', back_populates='licenses')
+    license_tier = db.relationship('LicenseTier', back_populates='licenses')
+    usages = db.relationship('LicenseUsage', order_by='LicenseUsage.period_start_date', back_populates='license')
+
+
+class LicenseTier(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(50), nullable=False)
+    version = db.Column(db.String(50), nullable=False)
+    start_date = db.Column(db.Date, nullable=False)
+    end_date = db.Column(db.Date, nullable=True)
+    basic_fee_d = db.Column(db.Float, nullable=True)
+    basic_fee_e = db.Column(db.Float, nullable=True)
+    max_storage_mb = db.Column(db.Integer, nullable=False)
+    additional_storage_price_d = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_storage_price_e = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_storage_bucket = db.Column(db.Integer, nullable=False)
+    included_embedding_mb = db.Column(db.Integer, nullable=False)
+    additional_embedding_price_d = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_embedding_price_e = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_embedding_bucket = db.Column(db.Integer, nullable=False)
+    included_interaction_tokens = db.Column(db.Integer, nullable=False)
+    additional_interaction_token_price_d = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_interaction_token_price_e = db.Column(db.Numeric(10, 4), nullable=False)
+    additional_interaction_bucket = db.Column(db.Integer, nullable=False)
+    standard_overage_embedding = db.Column(db.Float, nullable=False, default=0)
+    standard_overage_interaction = db.Column(db.Float, nullable=False, default=0)
+
+    licenses = db.relationship('License', back_populates='license_tier')
+
+
+class LicenseUsage(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    license_id = db.Column(db.Integer, db.ForeignKey('public.license.id'), nullable=False)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    storage_mb_used = db.Column(db.Float, default=0)
+    embedding_mb_used = db.Column(db.Float, default=0)
+    embedding_prompt_tokens_used = db.Column(db.Integer, default=0)
+    embedding_completion_tokens_used = db.Column(db.Integer, default=0)
+    embedding_total_tokens_used = db.Column(db.Integer, default=0)
+    interaction_prompt_tokens_used = db.Column(db.Integer, default=0)
+    interaction_completion_tokens_used = db.Column(db.Integer, default=0)
+    interaction_total_tokens_used = db.Column(db.Integer, default=0)
+    period_start_date = db.Column(db.Date, nullable=False)
+    period_end_date = db.Column(db.Date, nullable=False)
+
+    license = db.relationship('License', back_populates='usages')
+
+

common/models/user.py

@@ -1,8 +1,11 @@
+from datetime import date
+
 from common.extensions import db
 from flask_security import UserMixin, RoleMixin
 from sqlalchemy.dialects.postgresql import ARRAY
 import sqlalchemy as sa
-from sqlalchemy import CheckConstraint
+
+from common.models.entitlements import License
 
 
 class Tenant(db.Model):
@@ -21,6 +24,7 @@ class Tenant(db.Model):
     website = db.Column(db.String(255), nullable=True)
     timezone = db.Column(db.String(50), nullable=True, default='UTC')
     rag_context = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(20), nullable=True, server_default='Active')
 
     # language information
     default_language = db.Column(db.String(2), nullable=True)
@@ -30,40 +34,52 @@ class Tenant(db.Model):
     embedding_model = db.Column(db.String(50), nullable=True)
     llm_model = db.Column(db.String(50), nullable=True)
 
-    # Embedding variables
+    # # Embedding variables ==> To be removed once all migrations (dev + prod) have been done
-    html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
+    # html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
-    html_end_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'li'])
+    # html_end_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'li'])
-    html_included_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
+    # html_included_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
-    html_excluded_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
+    # html_excluded_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
-    html_excluded_classes = db.Column(ARRAY(sa.String(200)), nullable=True)
+    # html_excluded_classes = db.Column(ARRAY(sa.String(200)), nullable=True)
-
+    #
-    min_chunk_size = db.Column(db.Integer, nullable=True, default=2000)
+    # min_chunk_size = db.Column(db.Integer, nullable=True, default=2000)
-    max_chunk_size = db.Column(db.Integer, nullable=True, default=3000)
+    # max_chunk_size = db.Column(db.Integer, nullable=True, default=3000)
-
+    #
-    # Embedding search variables
+    # # Embedding search variables
-    es_k = db.Column(db.Integer, nullable=True, default=5)
+    # es_k = db.Column(db.Integer, nullable=True, default=5)
-    es_similarity_threshold = db.Column(db.Float, nullable=True, default=0.7)
+    # es_similarity_threshold = db.Column(db.Float, nullable=True, default=0.7)
-
+    #
-    # Chat variables
+    # # Chat variables
-    chat_RAG_temperature = db.Column(db.Float, nullable=True, default=0.3)
+    # chat_RAG_temperature = db.Column(db.Float, nullable=True, default=0.3)
-    chat_no_RAG_temperature = db.Column(db.Float, nullable=True, default=0.5)
+    # chat_no_RAG_temperature = db.Column(db.Float, nullable=True, default=0.5)
     fallback_algorithms = db.Column(ARRAY(sa.String(50)), nullable=True)
 
     # Licensing Information
-    license_start_date = db.Column(db.Date, nullable=True)
-    license_end_date = db.Column(db.Date, nullable=True)
-    allowed_monthly_interactions = db.Column(db.Integer, nullable=True)
     encrypted_chat_api_key = db.Column(db.String(500), nullable=True)
     encrypted_api_key = db.Column(db.String(500), nullable=True)
 
+    # # Tuning enablers
+    # embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
+    # rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
 
-    # Tuning enablers
+    # Entitlements
-    embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
+    currency = db.Column(db.String(20), nullable=True)
-    rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
+    usage_email = db.Column(db.String(255), nullable=True)
+    storage_dirty = db.Column(db.Boolean, nullable=True, default=False)
 
     # Relations
     users = db.relationship('User', backref='tenant')
     domains = db.relationship('TenantDomain', backref='tenant')
+    licenses = db.relationship('License', back_populates='tenant')
+    license_usages = db.relationship('LicenseUsage', backref='tenant')
+
+    @property
+    def current_license(self):
+        today = date.today()
+        return License.query.filter(
+            License.tenant_id == self.id,
+            License.start_date <= today,
+            (License.end_date.is_(None) | (License.end_date >= today))
+        ).order_by(License.start_date.desc()).first()
 
     def __repr__(self):
         return f"<Tenant {self.id}: {self.name}>"
@@ -75,27 +91,14 @@ class Tenant(db.Model):
             'website': self.website,
             'timezone': self.timezone,
             'rag_context': self.rag_context,
+            'type': self.type,
             'default_language': self.default_language,
             'allowed_languages': self.allowed_languages,
             'embedding_model': self.embedding_model,
             'llm_model': self.llm_model,
-            'html_tags': self.html_tags,
-            'html_end_tags': self.html_end_tags,
-            'html_included_elements': self.html_included_elements,
-            'html_excluded_elements': self.html_excluded_elements,
-            'html_excluded_classes': self.html_excluded_classes,
-            'min_chunk_size': self.min_chunk_size,
-            'max_chunk_size': self.max_chunk_size,
-            'es_k': self.es_k,
-            'es_similarity_threshold': self.es_similarity_threshold,
-            'chat_RAG_temperature': self.chat_RAG_temperature,
-            'chat_no_RAG_temperature': self.chat_no_RAG_temperature,
             'fallback_algorithms': self.fallback_algorithms,
-            'license_start_date': self.license_start_date,
+            'currency': self.currency,
-            'license_end_date': self.license_end_date,
+            'usage_email': self.usage_email,
-            'allowed_monthly_interactions': self.allowed_monthly_interactions,
-            'embed_tuning': self.embed_tuning,
-            'rag_tuning': self.rag_tuning,
         }
 
 

common/utils/business_event.py

@@ -0,0 +1,246 @@
+import os
+import uuid
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+from portkey_ai import Portkey, Config
+import logging
+
+from .business_event_context import BusinessEventContext
+from common.models.entitlements import BusinessEventLog
+from common.extensions import db
+
+
+class BusinessEvent:
+    # The BusinessEvent class itself is a context manager, but it doesn't use the @contextmanager decorator.
+    # Instead, it defines __enter__ and __exit__ methods explicitly. This is because we're doing something a bit more
+    # complex - we're interacting with the BusinessEventContext and the _business_event_stack.
+
+    def __init__(self, event_type: str, tenant_id: int, **kwargs):
+        self.event_type = event_type
+        self.tenant_id = tenant_id
+        self.trace_id = str(uuid.uuid4())
+        self.span_id = None
+        self.span_name = None
+        self.parent_span_id = None
+        self.document_version_id = kwargs.get('document_version_id')
+        self.document_version_file_size = kwargs.get('document_version_file_size')
+        self.chat_session_id = kwargs.get('chat_session_id')
+        self.interaction_id = kwargs.get('interaction_id')
+        self.environment = os.environ.get("FLASK_ENV", "development")
+        self.span_counter = 0
+        self.spans = []
+        self.llm_metrics = {
+            'total_tokens': 0,
+            'prompt_tokens': 0,
+            'completion_tokens': 0,
+            'total_time': 0,
+            'call_count': 0,
+            'interaction_type': None
+        }
+
+    def update_attribute(self, attribute: str, value: any):
+        if hasattr(self, attribute):
+            setattr(self, attribute, value)
+        else:
+            raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'")
+
+    def update_llm_metrics(self, metrics: dict):
+        self.llm_metrics['total_tokens'] += metrics['total_tokens']
+        self.llm_metrics['prompt_tokens'] += metrics['prompt_tokens']
+        self.llm_metrics['completion_tokens'] += metrics['completion_tokens']
+        self.llm_metrics['total_time'] += metrics['time_elapsed']
+        self.llm_metrics['call_count'] += 1
+        self.llm_metrics['interaction_type'] = metrics['interaction_type']
+
+    def reset_llm_metrics(self):
+        self.llm_metrics['total_tokens'] = 0
+        self.llm_metrics['prompt_tokens'] = 0
+        self.llm_metrics['completion_tokens'] = 0
+        self.llm_metrics['total_time'] = 0
+        self.llm_metrics['call_count'] = 0
+        self.llm_metrics['interaction_type'] = None
+
+    @contextmanager
+    def create_span(self, span_name: str):
+        # The create_span method is designed to be used as a context manager. We want to perform some actions when
+        # entering the span (like setting the span ID and name) and some actions when exiting the span (like removing
+        # these temporary attributes). The @contextmanager decorator allows us to write this method in a way that
+        # clearly separates the "entry" and "exit" logic, with the yield statement in between.
+
+        parent_span_id = self.span_id
+        self.span_counter += 1
+        new_span_id = str(uuid.uuid4())
+
+        # Save the current span info
+        self.spans.append((self.span_id, self.span_name, self.parent_span_id))
+
+        # Set the new span info
+        self.span_id = new_span_id
+        self.span_name = span_name
+        self.parent_span_id = parent_span_id
+
+        self.log(f"Starting span {span_name}")
+
+        try:
+            yield
+        finally:
+            if self.llm_metrics['call_count'] > 0:
+                self.log_final_metrics()
+                self.reset_llm_metrics()
+            self.log(f"Ending span {span_name}")
+            # Restore the previous span info
+            if self.spans:
+                self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+            else:
+                self.span_id = None
+                self.span_name = None
+                self.parent_span_id = None
+
+    def log(self, message: str, level: str = 'info'):
+        logger = logging.getLogger('business_events')
+        log_data = {
+            'event_type': self.event_type,
+            'tenant_id': self.tenant_id,
+            'trace_id': self.trace_id,
+            'span_id': self.span_id,
+            'span_name': self.span_name,
+            'parent_span_id': self.parent_span_id,
+            'document_version_id': self.document_version_id,
+            'document_version_file_size': self.document_version_file_size,
+            'chat_session_id': self.chat_session_id,
+            'interaction_id': self.interaction_id,
+            'environment': self.environment,
+        }
+        # log to Graylog
+        getattr(logger, level)(message, extra=log_data)
+
+        # Log to database
+        event_log = BusinessEventLog(
+            timestamp=dt.now(tz=tz.utc),
+            event_type=self.event_type,
+            tenant_id=self.tenant_id,
+            trace_id=self.trace_id,
+            span_id=self.span_id,
+            span_name=self.span_name,
+            parent_span_id=self.parent_span_id,
+            document_version_id=self.document_version_id,
+            document_version_file_size=self.document_version_file_size,
+            chat_session_id=self.chat_session_id,
+            interaction_id=self.interaction_id,
+            environment=self.environment,
+            message=message
+        )
+        db.session.add(event_log)
+        db.session.commit()
+
+    def log_llm_metrics(self, metrics: dict, level: str = 'info'):
+        self.update_llm_metrics(metrics)
+        message = "LLM Metrics"
+        logger = logging.getLogger('business_events')
+        log_data = {
+            'event_type': self.event_type,
+            'tenant_id': self.tenant_id,
+            'trace_id': self.trace_id,
+            'span_id': self.span_id,
+            'span_name': self.span_name,
+            'parent_span_id': self.parent_span_id,
+            'document_version_id': self.document_version_id,
+            'document_version_file_size': self.document_version_file_size,
+            'chat_session_id': self.chat_session_id,
+            'interaction_id': self.interaction_id,
+            'environment': self.environment,
+            'llm_metrics_total_tokens': metrics['total_tokens'],
+            'llm_metrics_prompt_tokens': metrics['prompt_tokens'],
+            'llm_metrics_completion_tokens': metrics['completion_tokens'],
+            'llm_metrics_total_time': metrics['time_elapsed'],
+            'llm_interaction_type': metrics['interaction_type'],
+        }
+        # log to Graylog
+        getattr(logger, level)(message, extra=log_data)
+
+        # Log to database
+        event_log = BusinessEventLog(
+            timestamp=dt.now(tz=tz.utc),
+            event_type=self.event_type,
+            tenant_id=self.tenant_id,
+            trace_id=self.trace_id,
+            span_id=self.span_id,
+            span_name=self.span_name,
+            parent_span_id=self.parent_span_id,
+            document_version_id=self.document_version_id,
+            document_version_file_size=self.document_version_file_size,
+            chat_session_id=self.chat_session_id,
+            interaction_id=self.interaction_id,
+            environment=self.environment,
+            llm_metrics_total_tokens=metrics['total_tokens'],
+            llm_metrics_prompt_tokens=metrics['prompt_tokens'],
+            llm_metrics_completion_tokens=metrics['completion_tokens'],
+            llm_metrics_total_time=metrics['time_elapsed'],
+            llm_interaction_type=metrics['interaction_type'],
+            message=message
+        )
+        db.session.add(event_log)
+        db.session.commit()
+
+    def log_final_metrics(self, level: str = 'info'):
+        logger = logging.getLogger('business_events')
+        message = "Final LLM Metrics"
+        log_data = {
+            'event_type': self.event_type,
+            'tenant_id': self.tenant_id,
+            'trace_id': self.trace_id,
+            'span_id': self.span_id,
+            'span_name': self.span_name,
+            'parent_span_id': self.parent_span_id,
+            'document_version_id': self.document_version_id,
+            'document_version_file_size': self.document_version_file_size,
+            'chat_session_id': self.chat_session_id,
+            'interaction_id': self.interaction_id,
+            'environment': self.environment,
+            'llm_metrics_total_tokens': self.llm_metrics['total_tokens'],
+            'llm_metrics_prompt_tokens': self.llm_metrics['prompt_tokens'],
+            'llm_metrics_completion_tokens': self.llm_metrics['completion_tokens'],
+            'llm_metrics_total_time': self.llm_metrics['total_time'],
+            'llm_metrics_call_count': self.llm_metrics['call_count'],
+            'llm_interaction_type': self.llm_metrics['interaction_type'],
+        }
+        # log to Graylog
+        getattr(logger, level)(message, extra=log_data)
+
+        # Log to database
+        event_log = BusinessEventLog(
+            timestamp=dt.now(tz=tz.utc),
+            event_type=self.event_type,
+            tenant_id=self.tenant_id,
+            trace_id=self.trace_id,
+            span_id=self.span_id,
+            span_name=self.span_name,
+            parent_span_id=self.parent_span_id,
+            document_version_id=self.document_version_id,
+            document_version_file_size=self.document_version_file_size,
+            chat_session_id=self.chat_session_id,
+            interaction_id=self.interaction_id,
+            environment=self.environment,
+            llm_metrics_total_tokens=self.llm_metrics['total_tokens'],
+            llm_metrics_prompt_tokens=self.llm_metrics['prompt_tokens'],
+            llm_metrics_completion_tokens=self.llm_metrics['completion_tokens'],
+            llm_metrics_total_time=self.llm_metrics['total_time'],
+            llm_metrics_call_count=self.llm_metrics['call_count'],
+            llm_interaction_type=self.llm_metrics['interaction_type'],
+            message=message
+        )
+        db.session.add(event_log)
+        db.session.commit()
+
+    def __enter__(self):
+        self.log(f'Starting Trace for {self.event_type}')
+        return BusinessEventContext(self).__enter__()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.llm_metrics['call_count'] > 0:
+            self.log_final_metrics()
+            self.reset_llm_metrics()
+        self.log(f'Ending Trace for {self.event_type}')
+        return BusinessEventContext(self).__exit__(exc_type, exc_val, exc_tb)

common/utils/business_event_context.py

@@ -0,0 +1,25 @@
+from werkzeug.local import LocalProxy, LocalStack
+
+_business_event_stack = LocalStack()
+
+
+def _get_current_event():
+    top = _business_event_stack.top
+    if top is None:
+        raise RuntimeError("No business event context found. Are you sure you're in a business event?")
+    return top
+
+
+current_event = LocalProxy(_get_current_event)
+
+
+class BusinessEventContext:
+    def __init__(self, event):
+        self.event = event
+
+    def __enter__(self):
+        _business_event_stack.push(self.event)
+        return self.event
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        _business_event_stack.pop()

common/utils/celery_utils.py

@@ -1,14 +1,16 @@
 from celery import Celery
 from kombu import Queue
 from werkzeug.local import LocalProxy
+from redbeat import RedBeatScheduler
 
 celery_app = Celery()
 
 
-def init_celery(celery, app):
+def init_celery(celery, app, is_beat=False):
     celery_app.main = app.name
     app.logger.debug(f'CELERY_BROKER_URL: {app.config["CELERY_BROKER_URL"]}')
     app.logger.debug(f'CELERY_RESULT_BACKEND: {app.config["CELERY_RESULT_BACKEND"]}')
+
     celery_config = {
         'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),
         'result_backend': app.config.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0'),
@@ -17,19 +19,40 @@ def init_celery(celery, app):
         'accept_content': app.config.get('CELERY_ACCEPT_CONTENT', ['json']),
         'timezone': app.config.get('CELERY_TIMEZONE', 'UTC'),
         'enable_utc': app.config.get('CELERY_ENABLE_UTC', True),
-        'task_routes': {'eveai_worker.tasks.create_embeddings': {'queue': 'embeddings',
-                                                                 'routing_key': 'embeddings.create_embeddings'}},
     }
+
+    if is_beat:
+        # Add configurations specific to Beat scheduler
+        celery_config['beat_scheduler'] = 'redbeat.RedBeatScheduler'
+        celery_config['redbeat_lock_key'] = 'redbeat::lock'
+        celery_config['beat_max_loop_interval'] = 10  # Adjust as needed
+
     celery_app.conf.update(**celery_config)
 
-    # Setting up Celery task queues
+    # Task queues for workers only
-    celery_app.conf.task_queues = (
+    if not is_beat:
-        Queue('default', routing_key='task.#'),
+        celery_app.conf.task_queues = (
-        Queue('embeddings', routing_key='embeddings.#', queue_arguments={'x-max-priority': 10}),
+            Queue('default', routing_key='task.#'),
-        Queue('llm_interactions', routing_key='llm_interactions.#', queue_arguments={'x-max-priority': 5}),
+            Queue('embeddings', routing_key='embeddings.#', queue_arguments={'x-max-priority': 10}),
-    )
+            Queue('llm_interactions', routing_key='llm_interactions.#', queue_arguments={'x-max-priority': 5}),
+            Queue('entitlements', routing_key='entitlements.#', queue_arguments={'x-max-priority': 10}),
+        )
+        celery_app.conf.task_routes = {
+            'eveai_workers.*': {  # All tasks from eveai_workers module
+                'queue': 'embeddings',
+                'routing_key': 'embeddings.#',
+            },
+            'eveai_chat_workers.*': {  # All tasks from eveai_chat_workers module
+                'queue': 'llm_interactions',
+                'routing_key': 'llm_interactions.#',
+            },
+            'eveai_entitlements.*': {  # All tasks from eveai_entitlements module
+                'queue': 'entitlements',
+                'routing_key': 'entitlements.#',
+            }
+        }
 
-    # Ensuring tasks execute with Flask application context
+    # Ensure tasks execute with Flask context
     class ContextTask(celery.Task):
         def __call__(self, *args, **kwargs):
             with app.app_context():
@@ -37,6 +60,39 @@ def init_celery(celery, app):
 
     celery.Task = ContextTask
 
+# Original init_celery before updating for beat
+# def init_celery(celery, app):
+#     celery_app.main = app.name
+#     app.logger.debug(f'CELERY_BROKER_URL: {app.config["CELERY_BROKER_URL"]}')
+#     app.logger.debug(f'CELERY_RESULT_BACKEND: {app.config["CELERY_RESULT_BACKEND"]}')
+#     celery_config = {
+#         'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),
+#         'result_backend': app.config.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0'),
+#         'task_serializer': app.config.get('CELERY_TASK_SERIALIZER', 'json'),
+#         'result_serializer': app.config.get('CELERY_RESULT_SERIALIZER', 'json'),
+#         'accept_content': app.config.get('CELERY_ACCEPT_CONTENT', ['json']),
+#         'timezone': app.config.get('CELERY_TIMEZONE', 'UTC'),
+#         'enable_utc': app.config.get('CELERY_ENABLE_UTC', True),
+#         'task_routes': {'eveai_worker.tasks.create_embeddings': {'queue': 'embeddings',
+#                                                                  'routing_key': 'embeddings.create_embeddings'}},
+#     }
+#     celery_app.conf.update(**celery_config)
+#
+#     # Setting up Celery task queues
+#     celery_app.conf.task_queues = (
+#         Queue('default', routing_key='task.#'),
+#         Queue('embeddings', routing_key='embeddings.#', queue_arguments={'x-max-priority': 10}),
+#         Queue('llm_interactions', routing_key='llm_interactions.#', queue_arguments={'x-max-priority': 5}),
+#     )
+#
+#     # Ensuring tasks execute with Flask application context
+#     class ContextTask(celery.Task):
+#         def __call__(self, *args, **kwargs):
+#             with app.app_context():
+#                 return self.run(*args, **kwargs)
+#
+#     celery.Task = ContextTask
+
 
 def make_celery(app_name, config):
     return celery_app

common/utils/cors_utils.py

@@ -23,6 +23,14 @@ def cors_after_request(response, prefix):
     current_app.logger.debug(f'request.args: {request.args}')
     current_app.logger.debug(f'request is json?: {request.is_json}')
 
+    # Exclude health checks from checks
+    if request.path.startswith('/healthz') or request.path.startswith('/_healthz'):
+        current_app.logger.debug('Skipping CORS headers for health checks')
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', '*')
+        response.headers.add('Access-Control-Allow-Methods', '*')
+        return response
+
     tenant_id = None
     allowed_origins = []
 

common/utils/document_utils.py

@@ -12,15 +12,17 @@ import requests
 from urllib.parse import urlparse, unquote
 import os
 from .eveai_exceptions import EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType
+from ..models.user import Tenant
 
 
 def create_document_stack(api_input, file, filename, extension, tenant_id):
     # Create the Document
-    new_doc = create_document(api_input, filename, tenant_id)
+    catalog_id = int(api_input.get('catalog_id'))
+    new_doc = create_document(api_input, filename, catalog_id)
     db.session.add(new_doc)
 
     # Create the DocumentVersion
-    new_doc_vers = create_version_for_document(new_doc,
+    new_doc_vers = create_version_for_document(new_doc, tenant_id,
                                                api_input.get('url', ''),
                                                api_input.get('language', 'en'),
                                                api_input.get('user_context', ''),
@@ -44,7 +46,7 @@ def create_document_stack(api_input, file, filename, extension, tenant_id):
     return new_doc, new_doc_vers
 
 
-def create_document(form, filename, tenant_id):
+def create_document(form, filename, catalog_id):
     new_doc = Document()
     if form['name'] == '':
         new_doc.name = filename.rsplit('.', 1)[0]
@@ -55,13 +57,13 @@ def create_document(form, filename, tenant_id):
         new_doc.valid_from = form['valid_from']
     else:
         new_doc.valid_from = dt.now(tz.utc)
-    new_doc.tenant_id = tenant_id
+    new_doc.catalog_id = catalog_id
     set_logging_information(new_doc, dt.now(tz.utc))
 
     return new_doc
 
 
-def create_version_for_document(document, url, language, user_context, user_metadata):
+def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata):
     new_doc_vers = DocumentVersion()
     if url != '':
         new_doc_vers.url = url
@@ -81,26 +83,30 @@ def create_version_for_document(document, url, language, user_context, user_meta
 
     set_logging_information(new_doc_vers, dt.now(tz.utc))
 
+    mark_tenant_storage_dirty(tenant_id)
+
     return new_doc_vers
 
 
 def upload_file_for_version(doc_vers, file, extension, tenant_id):
     doc_vers.file_type = extension
-    doc_vers.file_name = doc_vers.calc_file_name()
-    doc_vers.file_location = doc_vers.calc_file_location()
 
     # Normally, the tenant bucket should exist. But let's be on the safe side if a migration took place.
     minio_client.create_tenant_bucket(tenant_id)
 
     try:
-        minio_client.upload_document_file(
+        bn, on, size = minio_client.upload_document_file(
             tenant_id,
             doc_vers.doc_id,
             doc_vers.language,
             doc_vers.id,
-            doc_vers.file_name,
+            f"{doc_vers.id}.{extension}",
             file
         )
+        doc_vers.bucket_name = bn
+        doc_vers.object_name = on
+        doc_vers.file_size = size / 1048576  # Convert bytes to MB
+
         db.session.commit()
         current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for '
                                 f'document version {doc_vers.id} while uploading file.')
@@ -217,10 +223,9 @@ def process_multiple_urls(urls, tenant_id, api_input):
 
 
 def start_embedding_task(tenant_id, doc_vers_id):
-    task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
+    task = current_celery.send_task('create_embeddings',
-        tenant_id,
+                                    args=[tenant_id, doc_vers_id,],
-        doc_vers_id,
+                                    queue='embeddings')
-    ])
     current_app.logger.info(f'Embedding creation started for tenant {tenant_id}, '
                             f'Document Version {doc_vers_id}. '
                             f'Embedding creation task: {task.id}')
@@ -282,7 +287,7 @@ def edit_document_version(version_id, user_context):
         return None, str(e)
 
 
-def refresh_document_with_info(doc_id, api_input):
+def refresh_document_with_info(doc_id, tenant_id, api_input):
     doc = Document.query.get_or_404(doc_id)
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
@@ -290,11 +295,11 @@ def refresh_document_with_info(doc_id, api_input):
         return None, "This document has no URL. Only documents with a URL can be refreshed."
 
     new_doc_vers = create_version_for_document(
-        doc,
+        doc, tenant_id,
         old_doc_vers.url,
         api_input.get('language', old_doc_vers.language),
         api_input.get('user_context', old_doc_vers.user_context),
-        api_input.get('user_metadata', old_doc_vers.user_metadata)
+        api_input.get('user_metadata', old_doc_vers.user_metadata),
     )
 
     set_logging_information(new_doc_vers, dt.now(tz.utc))
@@ -314,18 +319,18 @@ def refresh_document_with_info(doc_id, api_input):
     response.raise_for_status()
     file_content = response.content
 
-    upload_file_for_version(new_doc_vers, file_content, extension, doc.tenant_id)
+    upload_file_for_version(new_doc_vers, file_content, extension, tenant_id)
 
-    task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
+    task = current_celery.send_task('create_embeddings', args=[tenant_id, new_doc_vers.id,], queue='embeddings')
-        doc.tenant_id,
+    current_app.logger.info(f'Embedding creation started for document {doc_id} on version {new_doc_vers.id} '
-        new_doc_vers.id,
+                            f'with task id: {task.id}.')
-    ])
 
     return new_doc_vers, task.id
 
 
 # Update the existing refresh_document function to use the new refresh_document_with_info
-def refresh_document(doc_id):
+def refresh_document(doc_id, tenant_id):
+    current_app.logger.info(f'Refreshing document {doc_id}')
     doc = Document.query.get_or_404(doc_id)
     old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first()
 
@@ -335,4 +340,11 @@ def refresh_document(doc_id):
         'user_metadata': old_doc_vers.user_metadata
     }
 
-    return refresh_document_with_info(doc_id, api_input)
+    return refresh_document_with_info(doc_id, tenant_id, api_input)
+
+
+# Function triggered when a document_version is created or updated
+def mark_tenant_storage_dirty(tenant_id):
+    tenant = db.session.query(Tenant).filter_by(id=int(tenant_id)).first()
+    tenant.storage_dirty = True
+    db.session.commit()

common/utils/eveai_exceptions.py

@@ -34,3 +34,10 @@ class EveAIUnsupportedFileType(EveAIException):
         super().__init__(message, status_code, payload)
 
 
+class EveAINoLicenseForTenant(EveAIException):
+    """Raised when no active license for a tenant is provided"""
+
+    def __init__(self, message="No license for tenant found", status_code=400, payload=None):
+        super().__init__(message, status_code, payload)
+
+

common/utils/minio_utils.py

@@ -50,13 +50,11 @@ class MinioClient:
             self.client.put_object(
                 bucket_name, object_name, io.BytesIO(file_data), len(file_data)
             )
-            return True
+            return bucket_name, object_name, len(file_data)
         except S3Error as err:
             raise Exception(f"Error occurred while uploading file: {err}")
 
-    def download_document_file(self, tenant_id, document_id, language, version_id, filename):
+    def download_document_file(self, tenant_id, bucket_name, object_name):
-        bucket_name = self.generate_bucket_name(tenant_id)
-        object_name = self.generate_object_name(document_id, language, version_id, filename)
         try:
             response = self.client.get_object(bucket_name, object_name)
             return response.read()

common/utils/model_utils.py

@@ -5,14 +5,19 @@ from flask import current_app
 from langchain_openai import OpenAIEmbeddings, ChatOpenAI
 from langchain_anthropic import ChatAnthropic
 from langchain_core.pydantic_v1 import BaseModel, Field
-from langchain.prompts import ChatPromptTemplate
+from typing import List, Any, Iterator
-import ast
+from collections.abc import MutableMapping
-from typing import List
 from openai import OpenAI
-# from groq import Groq
 from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL
+from portkey_ai.langchain.portkey_langchain_callback_handler import LangchainCallbackHandler
 
-from common.models.document import EmbeddingSmallOpenAI, EmbeddingLargeOpenAI
+from common.langchain.llm_metrics_handler import LLMMetricsHandler
+from common.langchain.tracked_openai_embeddings import TrackedOpenAIEmbeddings
+from common.langchain.tracked_transcribe import tracked_transcribe
+from common.models.document import EmbeddingSmallOpenAI, EmbeddingLargeOpenAI, Catalog
+from common.models.user import Tenant
+from config.model_config import MODEL_CONFIG
+from common.utils.business_event_context import current_event
 
 
 class CitedAnswer(BaseModel):
@@ -36,180 +41,205 @@ def set_language_prompt_template(cls, language_prompt):
     cls.__doc__ = language_prompt
 
 
-def select_model_variables(tenant):
+class ModelVariables(MutableMapping):
-    embedding_provider = tenant.embedding_model.rsplit('.', 1)[0]
+    def __init__(self, tenant: Tenant, catalog_id=None):
-    embedding_model = tenant.embedding_model.rsplit('.', 1)[1]
+        self.tenant = tenant
+        self.catalog_id = catalog_id
+        self._variables = self._initialize_variables()
+        self._embedding_model = None
+        self._llm = None
+        self._llm_no_rag = None
+        self._transcription_client = None
+        self._prompt_templates = {}
+        self._embedding_db_model = None
+        self.llm_metrics_handler = LLMMetricsHandler()
+        self._transcription_client = None
 
-    llm_provider = tenant.llm_model.rsplit('.', 1)[0]
+    def _initialize_variables(self):
-    llm_model = tenant.llm_model.rsplit('.', 1)[1]
+        variables = {}
 
-    # Set model variables
+        # Get the Catalog if catalog_id is passed
-    model_variables = {}
+        if self.catalog_id:
-    if tenant.es_k:
+            catalog = Catalog.query.get_or_404(self.catalog_id)
-        model_variables['k'] = tenant.es_k
-    else:
-        model_variables['k'] = 5
 
-    if tenant.es_similarity_threshold:
+            # We initialize the variables that are available knowing the tenant.
-        model_variables['similarity_threshold'] = tenant.es_similarity_threshold
+            variables['embed_tuning'] = catalog.embed_tuning or False
-    else:
-        model_variables['similarity_threshold'] = 0.7
 
-    if tenant.chat_RAG_temperature:
+            # Set HTML Chunking Variables
-        model_variables['RAG_temperature'] = tenant.chat_RAG_temperature
+            variables['html_tags'] = catalog.html_tags
-    else:
+            variables['html_end_tags'] = catalog.html_end_tags
-        model_variables['RAG_temperature'] = 0.3
+            variables['html_included_elements'] = catalog.html_included_elements
+            variables['html_excluded_elements'] = catalog.html_excluded_elements
+            variables['html_excluded_classes'] = catalog.html_excluded_classes
 
-    if tenant.chat_no_RAG_temperature:
+            # Set Chunk Size variables
-        model_variables['no_RAG_temperature'] = tenant.chat_no_RAG_temperature
+            variables['min_chunk_size'] = catalog.min_chunk_size
-    else:
+            variables['max_chunk_size'] = catalog.max_chunk_size
-        model_variables['no_RAG_temperature'] = 0.5
 
-    # Set Tuning variables
+        # Set the RAG Context (will have to change once specialists are defined
-    if tenant.embed_tuning:
+        variables['rag_context'] = self.tenant.rag_context or " "
-        model_variables['embed_tuning'] = tenant.embed_tuning
+        # Temporary setting until we have Specialists
-    else:
+        variables['rag_tuning'] = False
-        model_variables['embed_tuning'] = False
+        variables['RAG_temperature'] = 0.3
+        variables['no_RAG_temperature'] = 0.5
+        variables['k'] = 8
+        variables['similarity_threshold'] = 0.4
 
-    if tenant.rag_tuning:
+        # Set model providers
-        model_variables['rag_tuning'] = tenant.rag_tuning
+        variables['embedding_provider'], variables['embedding_model'] = self.tenant.embedding_model.rsplit('.', 1)
-    else:
+        variables['llm_provider'], variables['llm_model'] = self.tenant.llm_model.rsplit('.', 1)
-        model_variables['rag_tuning'] = False
+        variables["templates"] = current_app.config['PROMPT_TEMPLATES'][(f"{variables['llm_provider']}."
+                                                                         f"{variables['llm_model']}")]
+        current_app.logger.info(f"Loaded prompt templates: \n")
+        current_app.logger.info(f"{variables['templates']}")
 
-    if tenant.rag_context:
+        # Set model-specific configurations
-        model_variables['rag_context'] = tenant.rag_context
+        model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {})
-    else:
+        variables.update(model_config)
-        model_variables['rag_context'] = " "
 
-    # Set HTML Chunking Variables
+        variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][self.tenant.llm_model]
-    model_variables['html_tags'] = tenant.html_tags
-    model_variables['html_end_tags'] = tenant.html_end_tags
-    model_variables['html_included_elements'] = tenant.html_included_elements
-    model_variables['html_excluded_elements'] = tenant.html_excluded_elements
-    model_variables['html_excluded_classes'] = tenant.html_excluded_classes
 
-    # Set Chunk Size variables
+        if variables['tool_calling_supported']:
-    model_variables['min_chunk_size'] = tenant.min_chunk_size
+            variables['cited_answer_cls'] = CitedAnswer
-    model_variables['max_chunk_size'] = tenant.max_chunk_size
 
-    environment = os.getenv('FLASK_ENV', 'development')
+        variables['max_compression_duration'] = current_app.config['MAX_COMPRESSION_DURATION']
-    portkey_metadata = {'tenant_id': str(tenant.id), 'environment': environment}
+        variables['max_transcription_duration'] = current_app.config['MAX_TRANSCRIPTION_DURATION']
+        variables['compression_cpu_limit'] = current_app.config['COMPRESSION_CPU_LIMIT']
+        variables['compression_process_delay'] = current_app.config['COMPRESSION_PROCESS_DELAY']
 
-    # Set Embedding variables
+        return variables
-    match embedding_provider:
-        case 'openai':
-            portkey_headers = createHeaders(api_key=current_app.config.get('PORTKEY_API_KEY'),
-                                            provider='openai',
-                                            metadata=portkey_metadata)
-            match embedding_model:
-                case 'text-embedding-3-small':
-                    api_key = current_app.config.get('OPENAI_API_KEY')
-                    model_variables['embedding_model'] = OpenAIEmbeddings(api_key=api_key,
-                                                                          model='text-embedding-3-small',
-                                                                          base_url=PORTKEY_GATEWAY_URL,
-                                                                          default_headers=portkey_headers
-                                                                          )
-                    model_variables['embedding_db_model'] = EmbeddingSmallOpenAI
-                case 'text-embedding-3-large':
-                    api_key = current_app.config.get('OPENAI_API_KEY')
-                    model_variables['embedding_model'] = OpenAIEmbeddings(api_key=api_key,
-                                                                          model='text-embedding-3-large',
-                                                                          base_url=PORTKEY_GATEWAY_URL,
-                                                                          default_headers=portkey_headers
-                                                                          )
-                    model_variables['embedding_db_model'] = EmbeddingLargeOpenAI
-                case _:
-                    raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                                    f'error: Invalid embedding model')
-        case _:
-            raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                            f'error: Invalid embedding provider')
 
-    # Set Chat model variables
+    @property
-    match llm_provider:
+    def embedding_model(self):
-        case 'openai':
+        api_key = os.getenv('OPENAI_API_KEY')
-            portkey_headers = createHeaders(api_key=current_app.config.get('PORTKEY_API_KEY'),
+        model = self._variables['embedding_model']
-                                            metadata=portkey_metadata,
+        self._embedding_model = TrackedOpenAIEmbeddings(api_key=api_key,
-                                            provider='openai')
+                                                        model=model,
-            tool_calling_supported = False
+                                                        )
-            api_key = current_app.config.get('OPENAI_API_KEY')
+        self._embedding_db_model = EmbeddingSmallOpenAI \
-            model_variables['llm'] = ChatOpenAI(api_key=api_key,
+            if model == 'text-embedding-3-small' \
-                                                model=llm_model,
+            else EmbeddingLargeOpenAI
-                                                temperature=model_variables['RAG_temperature'],
-                                                base_url=PORTKEY_GATEWAY_URL,
-                                                default_headers=portkey_headers)
-            model_variables['llm_no_rag'] = ChatOpenAI(api_key=api_key,
-                                                       model=llm_model,
-                                                       temperature=model_variables['no_RAG_temperature'],
-                                                       base_url=PORTKEY_GATEWAY_URL,
-                                                       default_headers=portkey_headers)
-            tool_calling_supported = False
-            match llm_model:
-                case 'gpt-4o' | 'gpt-4o-mini':
-                    tool_calling_supported = True
-                    PDF_chunk_size = 10000
-                    PDF_chunk_overlap = 200
-                    PDF_min_chunk_size = 8000
-                    PDF_max_chunk_size = 12000
-                case _:
-                    raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                                    f'error: Invalid chat model')
-        case 'anthropic':
-            api_key = current_app.config.get('ANTHROPIC_API_KEY')
-            # Anthropic does not have the same 'generic' model names as OpenAI
-            llm_model_ext = current_app.config.get('ANTHROPIC_LLM_VERSIONS').get(llm_model)
-            model_variables['llm'] = ChatAnthropic(api_key=api_key,
-                                                   model=llm_model_ext,
-                                                   temperature=model_variables['RAG_temperature'])
-            model_variables['llm_no_rag'] = ChatAnthropic(api_key=api_key,
-                                                          model=llm_model_ext,
-                                                          temperature=model_variables['RAG_temperature'])
-            tool_calling_supported = True
-            PDF_chunk_size = 10000
-            PDF_chunk_overlap = 200
-            PDF_min_chunk_size = 8000
-            PDF_max_chunk_size = 12000
-        case _:
-            raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                            f'error: Invalid chat provider')
 
-    model_variables['PDF_chunk_size'] = PDF_chunk_size
+        return self._embedding_model
-    model_variables['PDF_chunk_overlap'] = PDF_chunk_overlap
-    model_variables['PDF_min_chunk_size'] = PDF_min_chunk_size
-    model_variables['PDF_max_chunk_size'] = PDF_max_chunk_size
 
-    if tool_calling_supported:
+    @property
-        model_variables['cited_answer_cls'] = CitedAnswer
+    def llm(self):
+        api_key = self.get_api_key_for_llm()
+        self._llm = ChatOpenAI(api_key=api_key,
+                               model=self._variables['llm_model'],
+                               temperature=self._variables['RAG_temperature'],
+                               callbacks=[self.llm_metrics_handler])
+        return self._llm
 
-    templates = current_app.config['PROMPT_TEMPLATES'][f'{llm_provider}.{llm_model}']
+    @property
-    model_variables['summary_template'] = templates['summary']
+    def llm_no_rag(self):
-    model_variables['rag_template'] = templates['rag']
+        api_key = self.get_api_key_for_llm()
-    model_variables['history_template'] = templates['history']
+        self._llm_no_rag = ChatOpenAI(api_key=api_key,
-    model_variables['encyclopedia_template'] = templates['encyclopedia']
+                                      model=self._variables['llm_model'],
-    model_variables['transcript_template'] = templates['transcript']
+                                      temperature=self._variables['RAG_temperature'],
-    model_variables['html_parse_template'] = templates['html_parse']
+                                      callbacks=[self.llm_metrics_handler])
-    model_variables['pdf_parse_template'] = templates['pdf_parse']
+        return self._llm_no_rag
 
-    model_variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][tenant.llm_model]
+    def get_api_key_for_llm(self):
+        if self._variables['llm_provider'] == 'openai':
+            api_key = os.getenv('OPENAI_API_KEY')
+        else:  # self._variables['llm_provider'] == 'anthropic'
+            api_key = os.getenv('ANTHROPIC_API_KEY')
 
-    # Transcription Client Variables.
+        return api_key
-    # Using Groq
-    # api_key = current_app.config.get('GROQ_API_KEY')
-    # model_variables['transcription_client'] = Groq(api_key=api_key)
-    # model_variables['transcription_model'] = 'whisper-large-v3'
 
-    # Using OpenAI for transcriptions
+    @property
-    portkey_metadata = {'tenant_id': str(tenant.id)}
+    def transcription_client(self):
-    portkey_headers = createHeaders(api_key=current_app.config.get('PORTKEY_API_KEY'),
+        api_key = os.getenv('OPENAI_API_KEY')
-                                    metadata=portkey_metadata,
+        self._transcription_client = OpenAI(api_key=api_key, )
-                                    provider='openai'
+        self._variables['transcription_model'] = 'whisper-1'
-                                    )
+        return self._transcription_client
-    api_key = current_app.config.get('OPENAI_API_KEY')
-    model_variables['transcription_client'] = OpenAI(api_key=api_key,
-                                                     base_url=PORTKEY_GATEWAY_URL,
-                                                     default_headers=portkey_headers)
-    model_variables['transcription_model'] = 'whisper-1'
 
+    def transcribe(self, *args, **kwargs):
+        return tracked_transcribe(self._transcription_client, *args, **kwargs)
+
+    @property
+    def embedding_db_model(self):
+        if self._embedding_db_model is None:
+            self._embedding_db_model = self.get_embedding_db_model()
+        return self._embedding_db_model
+
+    def get_embedding_db_model(self):
+        current_app.logger.debug("In get_embedding_db_model")
+        if self._embedding_db_model is None:
+            self._embedding_db_model = EmbeddingSmallOpenAI \
+                if self._variables['embedding_model'] == 'text-embedding-3-small' \
+                else EmbeddingLargeOpenAI
+        current_app.logger.debug(f"Embedding DB Model: {self._embedding_db_model}")
+        return self._embedding_db_model
+
+    def get_prompt_template(self, template_name: str) -> str:
+        current_app.logger.info(f"Getting prompt template for {template_name}")
+        if template_name not in self._prompt_templates:
+            self._prompt_templates[template_name] = self._load_prompt_template(template_name)
+        return self._prompt_templates[template_name]
+
+    def _load_prompt_template(self, template_name: str) -> str:
+        # In the future, this method will make an API call to Portkey
+        # For now, we'll simulate it with a placeholder implementation
+        # You can replace this with your current prompt loading logic
+        return self._variables['templates'][template_name]
+
+    def __getitem__(self, key: str) -> Any:
+        current_app.logger.debug(f"ModelVariables: Getting {key}")
+        # Support older template names (suffix = _template)
+        if key.endswith('_template'):
+            key = key[:-len('_template')]
+            current_app.logger.debug(f"ModelVariables: Getting modified {key}")
+        if key == 'embedding_model':
+            return self.embedding_model
+        elif key == 'embedding_db_model':
+            return self.embedding_db_model
+        elif key == 'llm':
+            return self.llm
+        elif key == 'llm_no_rag':
+            return self.llm_no_rag
+        elif key == 'transcription_client':
+            return self.transcription_client
+        elif key in self._variables.get('prompt_templates', []):
+            return self.get_prompt_template(key)
+        else:
+            value = self._variables.get(key)
+            if value is not None:
+                return value
+            else:
+                raise KeyError(f'Variable {key} does not exist in ModelVariables')
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._variables[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self._variables[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._variables)
+
+    def __len__(self):
+        return len(self._variables)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        return self.__getitem__(key) or default
+
+    def update(self, **kwargs) -> None:
+        self._variables.update(kwargs)
+
+    def items(self):
+        return self._variables.items()
+
+    def keys(self):
+        return self._variables.keys()
+
+    def values(self):
+        return self._variables.values()
+
+
+def select_model_variables(tenant, catalog_id=None):
+    model_variables = ModelVariables(tenant=tenant, catalog_id=catalog_id)
     return model_variables
 
 

common/utils/nginx_utils.py

@@ -6,7 +6,6 @@ def prefixed_url_for(endpoint, **values):
     prefix = request.headers.get('X-Forwarded-Prefix', '')
     scheme = request.headers.get('X-Forwarded-Proto', request.scheme)
     host = request.headers.get('Host', request.host)
-    current_app.logger.debug(f'prefix: {prefix}, scheme: {scheme}, host: {host}')
 
     external = values.pop('_external', False)
     generated_url = url_for(endpoint, **values)

common/utils/view_assistants.py

@@ -1,4 +1,4 @@
-from flask import flash
+from flask import flash, current_app
 
 
 def prepare_table(model_objects, column_names):
@@ -44,7 +44,8 @@ def form_validation_failed(request, form):
         for fieldName, errorMessages in form.errors.items():
             for err in errorMessages:
                 flash(f"Error in {fieldName}: {err}", 'danger')
+                current_app.logger.debug(f"Error in {fieldName}: {err}")
 
 
 def form_to_dict(form):
-    return {field.name: field.data for field in form if field.name != 'csrf_token' and hasattr(field, 'data')}
+    return {field.name: field.data for field in form if field.name != 'csrf_token' and hasattr(field, 'data')}

config/config.py

@@ -59,6 +59,9 @@ class Config(object):
     # supported languages
     SUPPORTED_LANGUAGES = ['en', 'fr', 'nl', 'de', 'es']
 
+    # supported currencies
+    SUPPORTED_CURRENCIES = ['€', '$']
+
     # supported LLMs
     SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed']
     SUPPORTED_LLMS = ['openai.gpt-4o', 'anthropic.claude-3-5-sonnet', 'openai.gpt-4o-mini']
@@ -137,9 +140,24 @@ class Config(object):
     MAIL_PASSWORD = environ.get('MAIL_PASSWORD')
     MAIL_DEFAULT_SENDER = ('eveAI Admin', MAIL_USERNAME)
 
+    # Langsmith settings
+    LANGCHAIN_TRACING_V2 = True
+    LANGCHAIN_ENDPOINT = 'https://api.smith.langchain.com'
+    LANGCHAIN_PROJECT = "eveai"
+
+
     SUPPORTED_FILE_TYPES = ['pdf', 'html', 'md', 'txt', 'mp3', 'mp4', 'ogg', 'srt']
 
+    TENANT_TYPES = ['Active', 'Demo', 'Inactive', 'Test']
 
+    # The maximum number of seconds allowed for audio compression (to save resources)
+    MAX_COMPRESSION_DURATION = 60*10    # 10 minutes
+    # The maximum number of seconds allowed for transcribing audio
+    MAX_TRANSCRIPTION_DURATION = 60*10  # 10 minutes
+    # Maximum CPU usage for a compression task
+    COMPRESSION_CPU_LIMIT = 50
+    # Delay between compressing chunks in seconds
+    COMPRESSION_PROCESS_DELAY = 1
 
 
 class DevConfig(Config):

config/gc_sa_eveai.json

@@ -1,13 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "eveai-420711",
-  "private_key_id": "e666408e75793321a6134243628346722a71b3a6",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaGTXCWpq08YD1\nOW4z+gncOlB7T/EIiEwsZgMp6pyUrNioGfiI9YN+uVR0nsUSmFf1YyerRgX7RqD5\nRc7T/OuX8iIvmloK3g7CaFezcVrjnBKcg/QsjDAt/OO3DTk4vykDlh/Kqxx73Jdv\nFH9YSV2H7ToWqIE8CTDnqe8vQS7Bq995c9fPlues31MgndRFg3CFkH0ldfZ4aGm3\n1RnBDyC+9SPQW9e7CJgNN9PWTmOT51Zyy5IRuV5OWePMQaGLVmCo5zNc/EHZEVRu\n1hxJPHL3NNmkYDY8tye8uHgjsAkv8QuwIuUSqnqjoo1/Yg+P0+9GCpePOAJRNxJS\n0YpDFWc5AgMBAAECggEACIU4/hG+bh97BD7JriFhfDDT6bg7g+pCs/hsAlxQ42jv\nOH7pyWuHJXGf5Cwx31usZAq4fcrgYnVpnyl8odIL628y9AjdI66wMuWhZnBFGJgK\nRhHcZWjW8nlXf0lBjwwFe4edzbn1AuWT5fYZ2HWDW2mthY/e8sUwqWPcWsjdifhz\nNR7V+Ia47McKXYgEKjyEObSP1NUOW24zH0DgxS52YPMwa1FoHn6+9Pr8P3TsTSO6\nh6f8tnd81DGl1UH4F5Bj/MHsQXyAMJbu44S4+rZ4Qlk+5xPp9hfCNpxWaHLIkJCg\nYXnC8UAjjyXiqyK0U0RjJf8TS1FxUI4iPepLNqp/pQKBgQDTicZnWFXmCFTnycWp\n66P3Yx0yvlKdUdfnoD/n9NdmUA3TZUlEVfb0IOm7ZFubF/zDTH87XrRiD/NVDbr8\n6bdhA1DXzraxhbfD36Hca6K74Ba4aYJsSWWwI0hL3FDSsv8c7qAIaUF2iwuHb7Y0\nRDcvZqowtQobcQC8cHLc/bI/ZwKBgQC6fMeGaU+lP6jhp9Nb/3Gz5Z1zzCu34IOo\nlgpTNZsowRKYLtjHifrEFi3XRxPKz5thMuJFniof5U4WoMYtRXy+PbgySvBpCia2\nXty05XssnLLMvLpYU5sbQvmOTe20zaIzLohRvvmqrydYIKu62NTubNeuD1L+Zr0q\nz1P5/wUgXwKBgQCW9MrRFQi3j1qHzkVwbOglsmUzwP3TpoQclw8DyIWuTZKQOMeA\nLJh+vr4NLCDzHLsT45MoGv0+vYM4PwQhV+e1I1idqLZXGMV60iv/0A/hYpjUIPch\nr38RoxwEhsRml7XWP7OUTQiaP7+Kdv3fbo6zFOB+wbLkwk90KgrOCX0aIQKBgFeK\n7esmErJjMPdFXk3om0q09nX+mWNHLOb+EDjBiGXYRM9V5oO9PQ/BzaEqh5sEXE+D\noH7H4cR5U3AB5yYnYYi41ngdf7//eO7Rl1AADhOCN9kum1eNX9mrVhU8deMTSRo3\ntNyTBwbeFF0lcRhUY5jNVW4rWW19cz3ed/B6i8CHAoGBAJ/l5rkV74Z5hg6BWNfQ\nYAg/4PLZmjnXIy5QdnWc/PYgbhn5+iVUcL9fSofFzJM1rjFnNcs3S90MGeOmfmo4\nM1WtcQFQbsCGt6+G5uEL/nf74mKUGpOqEM/XSkZ3inweWiDk3LK3iYfXCMBFouIr\n80IlzI1yMf7MVmWn3e1zPjCA\n-----END PRIVATE KEY-----\n",
-  "client_email": "eveai-349@eveai-420711.iam.gserviceaccount.com",
-  "client_id": "109927035346319712442",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/eveai-349%40eveai-420711.iam.gserviceaccount.com",
-  "universe_domain": "googleapis.com"
-}

config/logging_config.py

@@ -12,7 +12,12 @@ env = os.environ.get('FLASK_ENV', 'development')
 class CustomLogRecord(logging.LogRecord):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')  # Set default component value here
+        self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')
+
+    def __setattr__(self, name, value):
+        if name not in {'event_type', 'tenant_id', 'trace_id', 'span_id', 'span_name', 'parent_span_id',
+                        'document_version_id', 'chat_session_id', 'interaction_id', 'environment'}:
+            super().__setattr__(name, value)
 
 
 def custom_log_record_factory(*args, **kwargs):
@@ -32,7 +37,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_app.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -40,7 +45,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_workers.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -48,7 +53,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_chat.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -56,7 +61,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_chat_workers.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -64,7 +69,23 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_api.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
+            'backupCount': 10,
+            'formatter': 'standard',
+        },
+        'file_beat': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'logs/eveai_beat.log',
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
+            'backupCount': 10,
+            'formatter': 'standard',
+        },
+        'file_entitlements': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'logs/eveai_entitlements.log',
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -72,7 +93,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/sqlalchemy.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -80,7 +101,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/mailman.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -88,7 +109,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/security.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -96,7 +117,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/rag_tuning.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -104,7 +125,15 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/embed_tuning.log',
-            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
+            'backupCount': 10,
+            'formatter': 'standard',
+        },
+        'file_business_events': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'logs/business_events.log',
+            'maxBytes': 1024 * 1024 * 1,  # 1MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -159,6 +188,16 @@ LOGGING = {
             'level': 'DEBUG',
             'propagate': False
         },
+        'eveai_beat': {  # logger for the eveai_beat
+            'handlers': ['file_beat', 'graylog', ] if env == 'production' else ['file_beat', ],
+            'level': 'DEBUG',
+            'propagate': False
+        },
+        'eveai_entitlements': {  # logger for the eveai_entitlements
+            'handlers': ['file_entitlements', 'graylog', ] if env == 'production' else ['file_entitlements', ],
+            'level': 'DEBUG',
+            'propagate': False
+        },
         'sqlalchemy.engine': {  # logger for the sqlalchemy
             'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ],
             'level': 'DEBUG',
@@ -184,6 +223,11 @@ LOGGING = {
             'level': 'DEBUG',
             'propagate': False
         },
+        'business_events': {
+            'handlers': ['file_business_events', 'graylog'],
+            'level': 'DEBUG',
+            'propagate': False
+        },
         '': {  # root logger
             'handlers': ['console'],
             'level': 'WARNING',  # Set higher level for root to minimize noise

config/model_config.py

@@ -0,0 +1,41 @@
+MODEL_CONFIG = {
+    "openai": {
+        "gpt-4o": {
+            "tool_calling_supported": True,
+            "processing_chunk_size": 10000,
+            "processing_chunk_overlap": 200,
+            "processing_min_chunk_size": 8000,
+            "processing_max_chunk_size": 12000,
+            "prompt_templates": [
+                "summary", "rag", "history", "encyclopedia",
+                "transcript", "html_parse", "pdf_parse"
+            ]
+        },
+        "gpt-4o-mini": {
+            "tool_calling_supported": True,
+            "processing_chunk_size": 10000,
+            "processing_chunk_overlap": 200,
+            "processing_min_chunk_size": 8000,
+            "processing_max_chunk_size": 12000,
+            "prompt_templates": [
+                "summary", "rag", "history", "encyclopedia",
+                "transcript", "html_parse", "pdf_parse"
+            ]
+        },
+        # Add other OpenAI models here
+    },
+    "anthropic": {
+        "claude-3-5-sonnet": {
+            "tool_calling_supported": True,
+            "processing_chunk_size": 10000,
+            "processing_chunk_overlap": 200,
+            "processing_min_chunk_size": 8000,
+            "processing_max_chunk_size": 12000,
+            "prompt_templates": [
+                "summary", "rag", "history", "encyclopedia",
+                "transcript", "html_parse", "pdf_parse"
+            ]
+        },
+        # Add other Anthropic models here
+    },
+}

config/prompts/openai/gpt-4o.yaml

@@ -65,11 +65,13 @@ encyclopedia: |
 
 transcript: |
   You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
+  You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk.
+
 
   # Best practices and steps are:
   - Respect wordings and language(s) used in the transcription. Main language is {language}.
   - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.    
-  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part.
+  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences.
   - annotate the text to identify these logical parts using headings in {language}.
   - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.
 
@@ -77,4 +79,6 @@ transcript: |
 
   The transcript is between triple backquotes.
 
+  $$${previous_part}$$$
+
   ```{transcript}```

docker/build_and_push_eveai.sh

@@ -141,7 +141,7 @@ if [ $# -eq 0 ]; then
     SERVICES=()
     while IFS= read -r line; do
         SERVICES+=("$line")
-    done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)')
+    done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_|flower)')
 else
     SERVICES=("$@")
 fi
@@ -158,7 +158,7 @@ docker buildx use eveai_builder
 
 # Loop through services
 for SERVICE in "${SERVICES[@]}"; do
-    if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* ]]; then
+    if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* || "$SERVICE" == "flower" ]]; then
         if process_service "$SERVICE"; then
             echo "Successfully processed $SERVICE"
         else

docker/compose_dev.yaml

@@ -22,6 +22,8 @@ x-common-variables: &common-variables
   MAIL_PASSWORD: '$$6xsWGbNtx$$CFMQZqc*'
   MAIL_SERVER: mail.flow-it.net
   MAIL_PORT: 465
+  REDIS_URL: redis
+  REDIS_PORT: '6379'
   OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7'
   GROQ_API_KEY: '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'
   ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2'
@@ -32,6 +34,7 @@ x-common-variables: &common-variables
   MINIO_ACCESS_KEY: minioadmin
   MINIO_SECRET_KEY: minioadmin
   NGINX_SERVER_NAME: 'localhost http://macstudio.ask-eve-ai-local.com/'
+  LANGCHAIN_API_KEY: "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"
 
 
 networks:
@@ -96,12 +99,11 @@ services:
        minio:
          condition: service_healthy
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
+      test: ["CMD", "curl", "-f", "http://localhost:5001/healthz/ready"]
-      interval: 10s
+      interval: 30s
-      timeout: 5s
+      timeout: 1s
-      retries: 5
+      retries: 3
-#    entrypoint: ["scripts/entrypoint.sh"]
+      start_period: 30s
-#    command: ["scripts/start_eveai_app.sh"]
     networks:
       - eveai-network
 
@@ -113,8 +115,6 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_workers
@@ -132,13 +132,6 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
     networks:
       - eveai-network
 
@@ -168,12 +161,11 @@ services:
       redis:
         condition: service_healthy
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ]  # Adjust based on your health endpoint
+      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
-      interval: 10s
+      interval: 30s
-      timeout: 5s
+      timeout: 1s
-      retries: 5
+      retries: 3
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
+      start_period: 30s
-#    command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
     networks:
       - eveai-network
 
@@ -185,8 +177,6 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_chat_workers
@@ -202,13 +192,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
     networks:
       - eveai-network
 
@@ -240,15 +223,67 @@ services:
       minio:
         condition: service_healthy
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5003/health" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:5003/healthz/ready" ]
-      interval: 10s
+      interval: 30s
-      timeout: 5s
+      timeout: 1s
-      retries: 5
+      retries: 3
-    #    entrypoint: ["scripts/entrypoint.sh"]
+      start_period: 30s
-    #    command: ["scripts/start_eveai_api.sh"]
     networks:
       - eveai-network
 
+  eveai_beat:
+    image: josakola/eveai_beat:latest
+    build:
+      context: ..
+      dockerfile: ./docker/eveai_beat/Dockerfile
+      platforms:
+        - linux/amd64
+        - linux/arm64
+    environment:
+      <<: *common-variables
+      COMPONENT_NAME: eveai_beat
+    volumes:
+      - ../eveai_beat:/app/eveai_beat
+      - ../common:/app/common
+      - ../config:/app/config
+      - ../scripts:/app/scripts
+      - ../patched_packages:/app/patched_packages
+      - eveai_logs:/app/logs
+    depends_on:
+      redis:
+        condition: service_healthy
+    networks:
+      - eveai-network
+
+  eveai_entitlements:
+    image: josakola/eveai_entitlements:latest
+    build:
+      context: ..
+      dockerfile: ./docker/eveai_entitlements/Dockerfile
+      platforms:
+        - linux/amd64
+        - linux/arm64
+    environment:
+      <<: *common-variables
+      COMPONENT_NAME: eveai_entitlements
+    volumes:
+      - ../eveai_entitlements:/app/eveai_entitlements
+      - ../common:/app/common
+      - ../config:/app/config
+      - ../scripts:/app/scripts
+      - ../patched_packages:/app/patched_packages
+      - eveai_logs:/app/logs
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    networks:
+      - eveai-network
+
+
   db:
     hostname: db
     image: ankane/pgvector
@@ -285,6 +320,22 @@ services:
     networks:
       - eveai-network
 
+  flower:
+    image: josakola/flower:latest
+    build:
+      context: ..
+      dockerfile: ./docker/flower/Dockerfile
+    environment:
+      <<: *common-variables
+    volumes:
+      - ../scripts:/app/scripts
+    ports:
+      - "5555:5555"
+    depends_on:
+      - redis
+    networks:
+      - eveai-network
+
   minio:
     image: minio/minio
     ports:

docker/compose_stackhero.yaml

@@ -21,11 +21,13 @@ x-common-variables: &common-variables
   MAIL_USERNAME: 'evie_admin@askeveai.com'
   MAIL_PASSWORD: 's5D%R#y^v!s&6Z^i0k&'
   MAIL_SERVER: mail.askeveai.com
-  MAIL_PORT: 465
+  MAIL_PORT: '465'
   REDIS_USER: eveai
   REDIS_PASS: 'jHliZwGD36sONgbm0fc6SOpzLbknqq4RNF8K'
   REDIS_URL: 8bciqc.stackhero-network.com
   REDIS_PORT: '9961'
+  FLOWER_USER: 'Felucia'
+  FLOWER_PASSWORD: 'Jungles'
   OPENAI_API_KEY: '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'
   GROQ_API_KEY: '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'
   ANTHROPIC_API_KEY: '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'
@@ -38,6 +40,7 @@ x-common-variables: &common-variables
   MINIO_ACCESS_KEY: 04JKmQln8PQpyTmMiCPc
   MINIO_SECRET_KEY: 2PEZAD1nlpAmOyDV0TUTuJTQw1qVuYLF3A7GMs0D
   NGINX_SERVER_NAME: 'evie.askeveai.com mxz536.stackhero-network.com'
+  LANGCHAIN_API_KEY: "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"
 
 networks:
   eveai-network:
@@ -53,10 +56,6 @@ services:
     environment:
       <<: *common-variables
     volumes:
-#      - ../nginx:/etc/nginx
-#      - ../nginx/sites-enabled:/etc/nginx/sites-enabled
-#      - ../nginx/static:/etc/nginx/static
-#      - ../nginx/public:/etc/nginx/public
       - eveai_logs:/var/log/nginx
     labels:
       - "traefik.enable=true"
@@ -81,7 +80,7 @@ services:
     volumes:
       - eveai_logs:/app/logs
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
+      test: ["CMD", "curl", "-f", "http://localhost:5001/healthz/ready"]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -91,18 +90,11 @@ services:
   eveai_workers:
     platform: linux/amd64
     image: josakola/eveai_workers:latest
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_workers
     volumes:
       - eveai_logs:/app/logs
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
     networks:
       - eveai-network
 
@@ -117,7 +109,7 @@ services:
     volumes:
       - eveai_logs:/app/logs
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ]  # Adjust based on your health endpoint
+      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
       interval: 10s
       timeout: 5s
       retries: 5
@@ -127,18 +119,11 @@ services:
   eveai_chat_workers:
     platform: linux/amd64
     image: josakola/eveai_chat_workers:latest
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_chat_workers
     volumes:
       - eveai_logs:/app/logs
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
     networks:
       - eveai-network
 
@@ -153,20 +138,45 @@ services:
     volumes:
       - eveai_logs:/app/logs
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:5003/healthz/ready" ]
       interval: 10s
       timeout: 5s
       retries: 5
     networks:
       - eveai-network
 
+  eveai_beat:
+    platform: linux/amd64
+    image: josakola/eveai_beat:latest
+    environment:
+      <<: *common-variables
+      COMPONENT_NAME: eveai_beat
+    volumes:
+      - eveai_logs:/app/logs
+    networks:
+      - eveai-network
+
+  eveai_entitlements:
+    platform: linux/amd64
+    image: josakola/eveai_entitlements:latest
+    environment:
+      <<: *common-variables
+      COMPONENT_NAME: eveai_entitlements
+    volumes:
+      - eveai_logs:/app/logs
+    networks:
+      - eveai-network
+
+  flower:
+    image: josakola/flower:latest
+    environment:
+      <<: *common-variables
+    ports:
+      - "5555:5555"
+    networks:
+      - eveai-network
+
 volumes:
   eveai_logs:
-#  miniAre theo_data:
+
-#  db-data:
-#  redis-data:
-#  tenant-files:
-#secrets:
-#  db-password:
-#    file: ./db/password.txt
 

docker/eveai_api/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     gcc \
     postgresql-client \
+    curl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

docker/eveai_app/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     gcc \
     postgresql-client \
+    curl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

docker/eveai_beat/Dockerfile

@@ -0,0 +1,65 @@
+ARG PYTHON_VERSION=3.12.3
+FROM python:${PYTHON_VERSION}-slim as base
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+# Create directory for patched packages and set permissions
+RUN mkdir -p /app/patched_packages && \
+    chmod 777 /app/patched_packages
+
+# Ensure patches are applied to the application.
+ENV PYTHONPATH=/app/patched_packages:$PYTHONPATH
+
+WORKDIR /app
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/go/dockerfile-user-best-practices/
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/bin/bash" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+# Install necessary packages and build tools
+#RUN apt-get update && apt-get install -y \
+#    build-essential \
+#    gcc \
+#    && apt-get clean \
+#    && rm -rf /var/lib/apt/lists/*
+
+# Create logs directory and set permissions
+RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+
+# Install Python dependencies.
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+
+COPY requirements.txt /app/
+RUN python -m pip install -r /app/requirements.txt
+
+# Copy the source code into the container.
+COPY eveai_beat /app/eveai_beat
+COPY common /app/common
+COPY config /app/config
+COPY scripts /app/scripts
+COPY patched_packages /app/patched_packages
+COPY --chown=root:root scripts/entrypoint_no_db.sh /app/scripts/
+
+# Set ownership of the application directory to the non-privileged user
+RUN chown -R appuser:appuser /app
+
+# Set entrypoint and command
+ENTRYPOINT ["/app/scripts/entrypoint_no_db.sh"]
+CMD ["/app/scripts/start_eveai_beat.sh"]

docker/eveai_chat/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     gcc \
     postgresql-client \
+    curl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

docker/eveai_entitlements/Dockerfile

@@ -0,0 +1,69 @@
+ARG PYTHON_VERSION=3.12.3
+FROM python:${PYTHON_VERSION}-slim as base
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+# Create directory for patched packages and set permissions
+RUN mkdir -p /app/patched_packages && \
+    chmod 777 /app/patched_packages
+
+# Ensure patches are applied to the application.
+ENV PYTHONPATH=/app/patched_packages:$PYTHONPATH
+
+WORKDIR /app
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/go/dockerfile-user-best-practices/
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/bin/bash" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+# Install necessary packages and build tools
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    gcc \
+    postgresql-client \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create logs directory and set permissions
+RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+
+# Install Python dependencies.
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+
+COPY requirements.txt /app/
+RUN python -m pip install -r /app/requirements.txt
+
+# Copy the source code into the container.
+COPY eveai_entitlements /app/eveai_entitlements
+COPY common /app/common
+COPY config /app/config
+COPY scripts /app/scripts
+COPY patched_packages /app/patched_packages
+COPY --chown=root:root scripts/entrypoint.sh /app/scripts/
+
+# Set permissions for entrypoint script
+RUN chmod 777 /app/scripts/entrypoint.sh
+
+# Set ownership of the application directory to the non-privileged user
+RUN chown -R appuser:appuser /app
+
+# Set entrypoint and command
+ENTRYPOINT ["/app/scripts/entrypoint.sh"]
+CMD ["/app/scripts/start_eveai_entitlements.sh"]

docker/flower/Dockerfile

@@ -0,0 +1,34 @@
+ARG PYTHON_VERSION=3.12.3
+FROM python:${PYTHON_VERSION}-slim as base
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/bin/bash" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    gcc \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . /app
+COPY scripts/start_flower.sh /app/start_flower.sh
+RUN chmod a+x /app/start_flower.sh
+
+USER appuser
+
+CMD ["/app/start_flower.sh"]

docker/release_and_tag_eveai.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Initialize variables
+RELEASE_VERSION=""
+RELEASE_MESSAGE=""
+DOCKER_ACCOUNT="josakola"  # Your Docker account name
+
+# Parse input arguments
+while getopts r:m: flag
+do
+    case "${flag}" in
+        r) RELEASE_VERSION=${OPTARG};;
+        m) RELEASE_MESSAGE=${OPTARG};;
+        *)
+          echo "Usage: $0 -r <release_version> -m <release_message>"
+          exit 1 ;;
+    esac
+done
+
+# Ensure both version and message are provided
+if [ -z "$RELEASE_VERSION" ]; then
+  echo "Error: Release version not provided. Use -r <release_version>"
+  exit 1
+fi
+
+if [ -z "$RELEASE_MESSAGE" ]; then
+  echo "Error: Release message not provided. Use -m <release_message>"
+  exit 1
+fi
+
+# Path to your docker-compose file
+DOCKER_COMPOSE_FILE="compose_dev.yaml"
+
+# Get all the services defined in the docker-compose file
+SERVICES=$(docker-compose -f $DOCKER_COMPOSE_FILE config --services)
+
+# Tag and push images for all services that belong to your Docker account
+for SERVICE in $SERVICES; do
+    DOCKER_IMAGE="your-docker-repo/$SERVICE"
+
+    # Check if the image starts with your Docker account name
+    if [[ $DOCKER_IMAGE == $DOCKER_ACCOUNT* ]]; then
+        echo "Tagging Docker image for service: $SERVICE with version: $RELEASE_VERSION"
+
+        # Tag the 'latest' image with the new release version
+        docker tag $DOCKER_IMAGE:latest $DOCKER_IMAGE:$RELEASE_VERSION
+
+        # Push the newly tagged image to Docker Hub
+        docker push $DOCKER_IMAGE:$RELEASE_VERSION
+    else
+        echo "Skipping service: $SERVICE (not part of Docker account $DOCKER_ACCOUNT)"
+    fi
+done
+
+# Step 3: Tag the Git repository with the release version
+echo "Tagging Git repository with version: $RELEASE_VERSION"
+git tag -a v$RELEASE_VERSION -m "Release $RELEASE_VERSION: $RELEASE_MESSAGE"
+git push origin v$RELEASE_VERSION
+
+echo "Release process completed for version: $RELEASE_VERSION"

eveai_api/__init__.py

@@ -39,9 +39,12 @@ def create_app(config_file=None):
     # Register Necessary Extensions
     register_extensions(app)
 
-    # register Blueprints
+    # register Namespaces
     register_namespaces(api_rest)
 
+    # Register Blueprints
+    register_blueprints(app)
+
     # Error handler for the API
     @app.errorhandler(EveAIException)
     def handle_eveai_exception(error):
@@ -53,13 +56,6 @@ def create_app(config_file=None):
         app.logger.debug(f'Request URL: {request.url}')
         app.logger.debug(f'Request headers: {dict(request.headers)}')
 
-        # Log request arguments
-        app.logger.debug(f'Request args: {request.args}')
-
-        # Log form data if it's a POST request
-        if request.method == 'POST':
-            app.logger.debug(f'Form data: {request.form}')
-
         # Log JSON data if the content type is application/json
         if request.is_json:
             app.logger.debug(f'JSON data: {request.json}')
@@ -73,20 +69,28 @@ def create_app(config_file=None):
             app.logger.debug('Token request detected, skipping JWT verification')
             return
 
-        try:
+        # Check if this a health check request
-            verify_jwt_in_request(optional=True)
+        if request.path.startswith('/_healthz') or request.path.startswith('/healthz'):
-            tenant_id = get_jwt_identity()
+            app.logger.debug('Health check request detected, skipping JWT verification')
-            app.logger.debug(f'Tenant ID from JWT: {tenant_id}')
+        else:
+            try:
+                verify_jwt_in_request(optional=True)
+                tenant_id = get_jwt_identity()
+                app.logger.debug(f'Tenant ID from JWT: {tenant_id}')
 
-            if tenant_id:
+                if tenant_id:
-                Database(tenant_id).switch_schema()
+                    Database(tenant_id).switch_schema()
-                app.logger.debug(f'Switched to schema for tenant {tenant_id}')
+                    app.logger.debug(f'Switched to schema for tenant {tenant_id}')
-            else:
+                else:
-                app.logger.debug('No tenant ID found in JWT')
+                    app.logger.debug('No tenant ID found in JWT')
-        except Exception as e:
+            except Exception as e:
-            app.logger.error(f'Error in before_request: {str(e)}')
+                app.logger.error(f'Error in before_request: {str(e)}')
-            # Don't raise the exception here, let the request continue
+                # Don't raise the exception here, let the request continue
-            # The appropriate error handling will be done in the specific endpoints
+                # The appropriate error handling will be done in the specific endpoints
+
+    @app.route('/api/v1')
+    def swagger():
+        return api_rest.render_doc()
 
     return app
 
@@ -102,3 +106,9 @@ def register_extensions(app):
 def register_namespaces(app):
     api_rest.add_namespace(document_ns, path='/api/v1/documents')
     api_rest.add_namespace(auth_ns, path='/api/v1/auth')
+
+
+def register_blueprints(app):
+    from .views.healthz_views import healthz_bp
+    app.register_blueprint(healthz_bp)
+

eveai_api/api/document_api.py

@@ -33,6 +33,7 @@ document_ns = Namespace('documents', description='Document related operations')
 
 # Define models for request parsing and response serialization
 upload_parser = reqparse.RequestParser()
+upload_parser.add_argument('catalog_id', location='form', type=int, required=True, help='The catalog to add the file to')
 upload_parser.add_argument('file', location='files', type=FileStorage, required=True, help='The file to upload')
 upload_parser.add_argument('name', location='form', type=str, required=False, help='Name of the document')
 upload_parser.add_argument('language', location='form', type=str, required=True, help='Language of the document')
@@ -75,6 +76,7 @@ class AddDocument(Resource):
             validate_file_type(extension)
 
             api_input = {
+                'catalog_id': args.get('catalog_id'),
                 'name': args.get('name') or filename,
                 'language': args.get('language'),
                 'user_context': args.get('user_context'),
@@ -102,6 +104,7 @@ class AddDocument(Resource):
 
 # Models for AddURL
 add_url_model = document_ns.model('AddURL', {
+    'catalog_id': fields.Integer(required='True', description='ID of the catalog the URL needs to be added to'),
     'url': fields.String(required=True, description='URL of the document to add'),
     'name': fields.String(required=False, description='Name of the document'),
     'language': fields.String(required=True, description='Language of the document'),
@@ -138,6 +141,7 @@ class AddURL(Resource):
             file_content, filename, extension = process_url(args['url'], tenant_id)
 
             api_input = {
+                'catalog_id': args['catalog_id'],
                 'url': args['url'],
                 'name': args.get('name') or filename,
                 'language': args['language'],
@@ -213,7 +217,8 @@ class DocumentResource(Resource):
     @document_ns.response(200, 'Document refreshed successfully')
     def post(self, document_id):
         """Refresh a document"""
-        new_version, result = refresh_document(document_id)
+        tenant_id = get_jwt_identity()
+        new_version, result = refresh_document(document_id, tenant_id)
         if new_version:
             return {'message': f'Document refreshed. New version: {new_version.id}. Task ID: {result}'}, 200
         else:

eveai_api/views/healthz_views.py

@@ -0,0 +1,82 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+# Define Prometheus metrics
+api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint'])
+api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        # "celery": check_celery(),
+        "minio": check_minio(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('ping', queue='eveai_workers.ping')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+def check_minio():
+    try:
+        # List buckets to check if MinIO is accessible
+        minio_client.list_buckets()
+        return True
+    except Exception as e:
+        current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )

eveai_app/__init__.py

@@ -7,9 +7,11 @@ from werkzeug.middleware.proxy_fix import ProxyFix
 import logging.config
 
 from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session,
-                               minio_client, simple_encryption)
+                               minio_client, simple_encryption, metrics)
 from common.models.user import User, Role, Tenant, TenantDomain
 import common.models.interaction
+import common.models.entitlements
+import common.models.document
 from common.utils.nginx_utils import prefixed_url_for
 from config.logging_config import LOGGING
 from common.utils.security import set_tenant_session_data
@@ -114,10 +116,10 @@ def register_extensions(app):
     csrf.init_app(app)
     login_manager.init_app(app)
     cors.init_app(app)
-    # kms_client.init_app(app)
     simple_encryption.init_app(app)
     session.init_app(app)
     minio_client.init_app(app)
+    metrics.init_app(app)
 
 
 # Register Blueprints
@@ -132,3 +134,11 @@ def register_blueprints(app):
     app.register_blueprint(security_bp)
     from .views.interaction_views import interaction_bp
     app.register_blueprint(interaction_bp)
+    from .views.entitlements_views import entitlements_bp
+    app.register_blueprint(entitlements_bp)
+    from .views.administration_views import administration_bp
+    app.register_blueprint(administration_bp)
+    from .views.healthz_views import healthz_bp, init_healtz
+    app.register_blueprint(healthz_bp)
+    init_healtz(app)
+

eveai_app/templates/administration/trigger_actions.html

@@ -0,0 +1,22 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_selectable_table, render_pagination, render_field %}
+{% block title %}Trigger Actions{% endblock %}
+{% block content_title %}Trigger Actions{% endblock %}
+{% block content_description %}Manually trigger batch actions{% endblock %}
+{% block content %}
+
+<!-- Trigger action Form -->
+<form method="POST" action="{{ url_for('administration_bp.handle_trigger_action') }}">
+    <div class="form-group mt-3">
+        <button type="submit" name="action" value="update_usages" class="btn btn-secondary">Update Usages</button>
+    </div>
+</form>
+
+{% endblock %}
+
+{% block content_footer %}
+{% endblock %}
+
+{% block scripts %}
+{% endblock %}
+

eveai_app/templates/document/catalog.html

@@ -0,0 +1,23 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_field %}
+
+{% block title %}Catalog Registration{% endblock %}
+
+{% block content_title %}Register Catalog{% endblock %}
+{% block content_description %}Define a new catalog of documents in Evie's Library{% endblock %}
+
+{% block content %}
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {%  set disabled_fields = [] %}
+        {%  set exclude_fields = [] %}
+        {% for field in form %}
+            {{ render_field(field, disabled_fields, exclude_fields) }}
+        {% endfor %}
+        <button type="submit" class="btn btn-primary">Register Catalog</button>
+    </form>
+{% endblock %}
+
+{% block content_footer %}
+
+{% endblock %}

eveai_app/templates/document/catalogs.html

@@ -0,0 +1,24 @@
+{% extends 'base.html' %}
+{%  from 'macros.html' import render_selectable_table, render_pagination %}
+
+{% block title %}Documents{% endblock %}
+
+{% block content_title %}Catalogs{% endblock %}
+{% block content_description %}View Catalogs for Tenant{% endblock %}
+{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
+
+{% block content %}
+<div class="container">
+    <form method="POST" action="{{ url_for('document_bp.handle_catalog_selection') }}">
+        {{ render_selectable_table(headers=["Catalog ID", "Name"], rows=rows, selectable=True, id="catalogsTable") }}
+        <div class="form-group mt-3">
+            <button type="submit" name="action" value="set_session_catalog" class="btn btn-primary">Set Session Catalog</button>
+            <button type="submit" name="action" value="edit_catalog" class="btn btn-primary">Edit Catalog</button>
+        </div>
+    </form>
+</div>
+{% endblock %}
+
+{% block content_footer %}
+    {{ render_pagination(pagination, 'document_bp.catalogs') }}
+{% endblock %}

eveai_app/templates/document/document_versions.html

@@ -10,7 +10,7 @@
 {% block content %}
 <div class="container">
     <form method="POST" action="{{ url_for('document_bp.handle_document_version_selection') }}">
-        {{ render_selectable_table(headers=["ID", "URL", "File Loc.", "File Name", "File Type", "Process.", "Proces. Start", "Proces. Finish", "Proces. Error"], rows=rows, selectable=True, id="versionsTable") }}
+        {{ render_selectable_table(headers=["ID", "URL", "Object Name", "File Type", "Process.", "Proces. Start", "Proces. Finish", "Proces. Error"], rows=rows, selectable=True, id="versionsTable") }}
         <div class="form-group mt-3">
             <button type="submit" name="action" value="edit_document_version" class="btn btn-primary">Edit Document Version</button>
             <button type="submit" name="action" value="process_document_version" class="btn btn-danger">Process Document Version</button>

eveai_app/templates/document/document_versions_list_view.html

@@ -23,15 +23,23 @@
 
     {{ render_collapsible_section('Filter', 'Filter Options', filter_form) }}
 
-    <!-- Document Versions Table -->
+    <div class="form-group mt-3">
-    {{ render_selectable_sortable_table(
+        <form method="POST" action="{{ url_for('document_bp.handle_document_version_selection') }}">
-        headers=["ID", "File Type", "Processing", "Processing Start", "Processing Finish", "Processing Error"],
+            <!-- Document Versions Table -->
-        rows=rows,
+            {{ render_selectable_sortable_table(
-        selectable=True,
+                headers=["ID", "File Type", "Processing", "Processing Start", "Processing Finish", "Processing Error"],
-        id="documentVersionsTable",
+                rows=rows,
-        sort_by=sort_by,
+                selectable=True,
-        sort_order=sort_order
+                id="documentVersionsTable",
-    ) }}
+                sort_by=sort_by,
+                sort_order=sort_order
+            ) }}
+            <div class="form-group mt-4">
+                <button type="submit" name="action" value="edit_document_version" class="btn btn-primary">Edit Document Version</button>
+                <button type="submit" name="action" value="process_document_version" class="btn btn-danger">Process Document Version</button>
+            </div>
+        </form>
+    </div>
 {% endblock %}
 
 {% block content_footer %}

eveai_app/templates/document/documents.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-{%  from 'macros.html' import render_selectable_table, render_pagination %}
+{% from 'macros.html' import render_selectable_table, render_pagination, render_filter_field, render_date_filter_field, render_collapsible_section, render_selectable_sortable_table_with_dict_headers %}
 
 {% block title %}Documents{% endblock %}
 
@@ -8,18 +8,88 @@
 {% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
 
 {% block content %}
-<div class="container">
+    <!-- Filter Form -->
-    <form method="POST" action="{{ url_for('document_bp.handle_document_selection') }}">
+    {% set filter_form %}
-        {{ render_selectable_table(headers=["Document ID", "Name", "Valid From", "Valid To"], rows=rows, selectable=True, id="documentsTable") }}
+        <form method="GET" action="{{ url_for('document_bp.documents') }}">
-        <div class="form-group mt-3">
+            {{ render_filter_field('catalog_id', 'Catalog', filter_options['catalog_id'], filters.get('catalog_id', [])) }}
-            <button type="submit" name="action" value="edit_document" class="btn btn-primary">Edit Document</button>
+            {{ render_filter_field('validity', 'Validity', filter_options['validity'], filters.get('validity', [])) }}
-            <button type="submit" name="action" value="document_versions" class="btn btn-secondary">Show Document Versions</button>
+
-            <button type="submit" name="action" value="refresh_document" class="btn btn-secondary">Refresh Document (new version)</button>
+            <button type="submit" class="btn btn-primary">Apply Filters</button>
-        </div>
+        </form>
-    </form>
+    {% endset %}
-</div>
+
+    {{ render_collapsible_section('Filter', 'Filter Options', filter_form) }}
+
+    <div class="form-group mt-3">
+        <form method="POST" action="{{ url_for('document_bp.handle_document_selection') }}">
+            <!-- Documents Table -->
+            {{ render_selectable_sortable_table_with_dict_headers(
+                headers=[
+                    {"text": "ID", "sort": "id"},
+                    {"text": "Name", "sort": "name"},
+                    {"text": "Catalog", "sort": "catalog_name"},
+                    {"text": "Valid From", "sort": "valid_from"},
+                    {"text": "Valid To", "sort": "valid_to"}
+                ],
+                rows=rows,
+                selectable=True,
+                id="documentsTable",
+                sort_by=sort_by,
+                sort_order=sort_order
+            ) }}
+            <div class="form-group mt-4">
+                <button type="submit" name="action" value="edit_document" class="btn btn-primary">Edit Document</button>
+                <button type="submit" name="action" value="document_versions" class="btn btn-secondary">Show Document Versions</button>
+                <button type="submit" name="action" value="refresh_document" class="btn btn-secondary">Refresh Document (new version)</button>
+            </div>
+        </form>
+    </div>
 {% endblock %}
 
 {% block content_footer %}
     {{ render_pagination(pagination, 'document_bp.documents') }}
+{% endblock %}
+
+{% block scripts %}
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    const table = document.getElementById('documentsTable');
+    const headers = table.querySelectorAll('th.sortable');
+
+    headers.forEach(header => {
+        header.addEventListener('click', function() {
+            const sortBy = this.dataset.sort;
+            let sortOrder = 'asc';
+
+            if (this.querySelector('.fa-sort-up')) {
+                sortOrder = 'desc';
+            } else if (this.querySelector('.fa-sort-down')) {
+                sortOrder = 'none';
+            }
+
+            window.location.href = updateQueryStringParameter(window.location.href, 'sort_by', sortBy);
+            window.location.href = updateQueryStringParameter(window.location.href, 'sort_order', sortOrder);
+        });
+    });
+
+    function updateQueryStringParameter(uri, key, value) {
+        var re = new RegExp("([?&])" + key + "=.*?(&|$)", "i");
+        var separator = uri.indexOf('?') !== -1 ? "&" : "?";
+        if (uri.match(re)) {
+            return uri.replace(re, '$1' + key + "=" + value + '$2');
+        }
+        else {
+            return uri + separator + key + "=" + value;
+        }
+    }
+
+    table.addEventListener('change', function(event) {
+        if (event.target.type === 'radio') {
+            var selectedRow = event.target.closest('tr');
+            var documentId = selectedRow.cells[1].textContent;
+            console.log('Selected Document ID:', documentId);
+        }
+    });
+});
+</script>
 {% endblock %}

eveai_app/templates/document/edit_catalog.html

@@ -0,0 +1,25 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_field %}
+
+{% block title %}Edit Catalog{% endblock %}
+
+{% block content_title %}Edit Catalog{% endblock %}
+{% block content_description %}Edit a catalog of documents in Evie's Library.
+When you change chunking of embedding information, you'll need to manually refresh the library if you want immediate impact.
+{% endblock %}
+
+{% block content %}
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {%  set disabled_fields = [] %}
+        {%  set exclude_fields = [] %}
+        {% for field in form %}
+            {{ render_field(field, disabled_fields, exclude_fields) }}
+        {% endfor %}
+        <button type="submit" class="btn btn-primary">Register Catalog</button>
+    </form>
+{% endblock %}
+
+{% block content_footer %}
+
+{% endblock %}

eveai_app/templates/document/edit_document.html

@@ -8,11 +8,17 @@
 {% block content %}
     <form method="post">
         {{ form.hidden_tag() }}
-        {%  set disabled_fields = [] %}
+        {% set disabled_fields = [] %}
-        {%  set exclude_fields = [] %}
+        {% set exclude_fields = [] %}
-        {% for field in form %}
+
-            {{ render_field(field, disabled_fields, exclude_fields) }}
+        {{ render_field(form.name, disabled_fields, exclude_fields) }}
-        {% endfor %}
+        {{ render_field(form.valid_from, disabled_fields, exclude_fields) }}
+        {{ render_field(form.valid_to, disabled_fields, exclude_fields) }}
+
+        <div class="form-group">
+            <label for="catalog_name">Catalog</label>
+            <input type="text" class="form-control" id="catalog_name" value="{{ catalog_name }}" readonly>
+        </div>
         <button type="submit" class="btn btn-primary">Update Document</button>
     </form>
-{% endblock %}
+{% endblock %}

eveai_app/templates/entitlements/edit_license.html

@@ -0,0 +1,71 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_field, render_included_field %}
+
+{% block title %}Edit License for Current Tenant{% endblock %}
+
+{% block content_title %}Edit License for Current Tenant{% endblock %}
+{% block content_description %}Edit a License based on the selected License Tier for the current Tenant{% endblock %}
+
+{% block content %}
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {% set main_fields = ['start_date', 'end_date', 'currency', 'yearly_payment', 'basic_fee'] %}
+        {% for field in form %}
+            {{ render_included_field(field, disabled_fields=['currency'], include_fields=main_fields) }}
+        {% endfor %}
+        <!-- Nav Tabs -->
+        <div class="row mt-5">
+            <div class="col-lg-12">
+                <div class="nav-wrapper position-relative end-0">
+                    <ul class="nav nav-pills nav-fill p-1" role="tablist">
+                        <li class="nav-item" role="presentation">
+                            <a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#storage-tab" role="tab" aria-controls="model-info" aria-selected="true">
+                                Storage
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-tab" role="tab" aria-controls="license-info" aria-selected="false">
+                                Embedding
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#interaction-tab" role="tab" aria-controls="chunking" aria-selected="false">
+                                Interaction
+                            </a>
+                        </li>
+                    </ul>
+                </div>
+                <div class="tab-content tab-space">
+                    <!-- Storage Tab -->
+                    <div class="tab-pane fade show active" id="storage-tab" role="tabpanel">
+                        {% set storage_fields = ['max_storage_tokens', 'additional_storage_token_price', 'additional_storage_bucket'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=storage_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- Embedding Tab -->
+                    <div class="tab-pane fade" id="embedding-tab" role="tabpanel">
+                        {% set embedding_fields = ['included_embedding_tokens', 'additional_embedding_token_price', 'additional_embedding_bucket'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=embedding_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- Interaction Tab -->
+                    <div class="tab-pane fade" id="interaction-tab" role="tabpanel">
+                        {% set interaction_fields = ['included_interaction_tokens', 'additional_interaction_token_price', 'additional_interaction_bucket'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=interaction_fields) }}
+                        {% endfor %}
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <button type="submit" class="btn btn-primary">Save License</button>
+    </form>
+{% endblock %}
+
+
+{% block content_footer %}
+
+{% endblock %}

eveai_app/templates/entitlements/license.html

@@ -0,0 +1,71 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_field, render_included_field %}
+
+{% block title %}Create or Edit License for Current Tenant{% endblock %}
+
+{% block content_title %}Create or Edit License for Current Tenant{% endblock %}
+{% block content_description %}Create or Edit a new License based on the selected License Tier for the current Tenant{% endblock %}
+
+{% block content %}
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {% set main_fields = ['start_date', 'end_date', 'currency', 'yearly_payment', 'basic_fee'] %}
+        {% for field in form %}
+            {{ render_included_field(field, disabled_fields=ext_disabled_fields + ['currency'], include_fields=main_fields) }}
+        {% endfor %}
+        <!-- Nav Tabs -->
+        <div class="row mt-5">
+            <div class="col-lg-12">
+                <div class="nav-wrapper position-relative end-0">
+                    <ul class="nav nav-pills nav-fill p-1" role="tablist">
+                        <li class="nav-item" role="presentation">
+                            <a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#storage-tab" role="tab" aria-controls="model-info" aria-selected="true">
+                                Storage
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-tab" role="tab" aria-controls="license-info" aria-selected="false">
+                                Embedding
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#interaction-tab" role="tab" aria-controls="chunking" aria-selected="false">
+                                Interaction
+                            </a>
+                        </li>
+                    </ul>
+                </div>
+                <div class="tab-content tab-space">
+                    <!-- Storage Tab -->
+                    <div class="tab-pane fade show active" id="storage-tab" role="tabpanel">
+                        {% set storage_fields = ['max_storage_mb', 'additional_storage_price', 'additional_storage_bucket'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=ext_disabled_fields, include_fields=storage_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- Embedding Tab -->
+                    <div class="tab-pane fade" id="embedding-tab" role="tabpanel">
+                        {% set embedding_fields = ['included_embedding_mb', 'additional_embedding_price', 'additional_embedding_bucket', 'overage_embedding'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=ext_disabled_fields, include_fields=embedding_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- Interaction Tab -->
+                    <div class="tab-pane fade" id="interaction-tab" role="tabpanel">
+                        {% set interaction_fields = ['included_interaction_tokens', 'additional_interaction_token_price', 'additional_interaction_bucket', 'overage_interaction'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=ext_disabled_fields, include_fields=interaction_fields) }}
+                        {% endfor %}
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <button type="submit" class="btn btn-primary">Save License</button>
+    </form>
+{% endblock %}
+
+
+{% block content_footer %}
+
+{% endblock %}

eveai_app/templates/entitlements/license_tier.html

@@ -0,0 +1,71 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_field, render_included_field %}
+
+{% block title %}Register or Edit License Tier{% endblock %}
+
+{% block content_title %}Register or Edit License Tier{% endblock %}
+{% block content_description %}Register or Edit License Tier{% endblock %}
+
+{% block content %}
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {% set main_fields = ['name', 'version', 'start_date', 'end_date', 'basic_fee_d', 'basic_fee_e'] %}
+        {% for field in form %}
+            {{ render_included_field(field, disabled_fields=[], include_fields=main_fields) }}
+        {% endfor %}
+        <!-- Nav Tabs -->
+        <div class="row mt-5">
+            <div class="col-lg-12">
+                <div class="nav-wrapper position-relative end-0">
+                    <ul class="nav nav-pills nav-fill p-1" role="tablist">
+                        <li class="nav-item" role="presentation">
+                            <a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#storage-tab" role="tab" aria-controls="model-info" aria-selected="true">
+                                Storage
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-tab" role="tab" aria-controls="license-info" aria-selected="false">
+                                Embedding
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#interaction-tab" role="tab" aria-controls="chunking" aria-selected="false">
+                                Interaction
+                            </a>
+                        </li>
+                    </ul>
+                </div>
+                <div class="tab-content tab-space">
+                    <!-- Storage Tab -->
+                    <div class="tab-pane fade show active" id="storage-tab" role="tabpanel">
+                        {% set storage_fields = ['max_storage_mb', 'additional_storage_price_d', 'additional_storage_price_e', 'additional_storage_bucket'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=storage_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- Embedding Tab -->
+                    <div class="tab-pane fade" id="embedding-tab" role="tabpanel">
+                        {% set embedding_fields = ['included_embedding_mb', 'additional_embedding_price_d', 'additional_embedding_price_e', 'additional_embedding_bucket', 'standard_overage_embedding'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=embedding_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- Interaction Tab -->
+                    <div class="tab-pane fade" id="interaction-tab" role="tabpanel">
+                        {% set interaction_fields = ['included_interaction_tokens', 'additional_interaction_token_price_d', 'additional_interaction_token_price_e', 'additional_interaction_bucket', 'standard_overage_interaction'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=interaction_fields) }}
+                        {% endfor %}
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <button type="submit" class="btn btn-primary">Save License Tier</button>
+    </form>
+{% endblock %}
+
+
+{% block content_footer %}
+
+{% endblock %}

eveai_app/templates/entitlements/view_license_tiers.html

@@ -0,0 +1,24 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_selectable_table, render_pagination, render_field %}
+{% block title %}License Tier Selection{% endblock %}
+{% block content_title %}Select a License Tier{% endblock %}
+{% block content_description %}Select a License Tier to continue{% endblock %}
+{% block content %}
+
+<!-- License Tier Selection Form -->
+<form method="POST" action="{{ url_for('entitlements_bp.handle_license_tier_selection') }}">
+    {{ render_selectable_table(headers=["ID", "Name", "Version", "Start Date", "End Date"], rows=rows, selectable=True, id="licenseTierTable") }}
+    <div class="form-group mt-3">
+        <button type="submit" name="action" value="edit_license_tier" class="btn btn-primary">Edit License Tier</button>
+        <button type="submit" name="action" value="create_license_for_tenant" class="btn btn-secondary">Create License for Current Tenant</button>
+    </div>
+</form>
+
+{% endblock %}
+
+{% block content_footer %}
+{{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{% endblock %}
+
+
+

eveai_app/templates/entitlements/view_usages.html

@@ -0,0 +1,28 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_selectable_table, render_pagination %}
+
+{% block title %}View License Usage{% endblock %}
+
+{%  block content_title %}View License Usage{% endblock %}
+{% block content_description %}View License Usage{% endblock %}
+
+{% block content %}
+<form action="{{ url_for('user_bp.handle_user_action') }}" method="POST">
+    {{ render_selectable_table(headers=["Usage ID", "Start Date", "End Date", "Storage (MiB)", "Embedding (MiB)", "Interaction (tokens)"], rows=rows, selectable=False, id="usagesTable") }}
+<!--    <div class="form-group mt-3">-->
+<!--        <button type="submit" name="action" value="edit_user" class="btn btn-primary">Edit Selected User</button>-->
+<!--        <button type="submit" name="action" value="resend_confirmation_email" class="btn btn-secondary">Resend Confirmation Email</button>-->
+<!--        <button type="submit" name="action" value="send_password_reset_email" class="btn btn-secondary">Send Password Reset Email</button>-->
+<!--        <button type="submit" name="action" value="reset_uniquifier" class="btn btn-secondary">Reset Uniquifier</button>-->
+<!--        &lt;!&ndash; Additional buttons can be added here for other actions &ndash;&gt;-->
+<!--    </div>-->
+</form>
+{% endblock %}
+
+{% block content_footer %}
+    {{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{% endblock %}
+
+{% block scripts %}
+
+{% endblock %}

eveai_app/templates/header.html

@@ -1,5 +1,5 @@
 <header class="header-2">
-    <div class="page-header min-vh-25" style="background-image: url({{url_for('static', filename='/assets/img/EveAI_bg.jpg')}})" loading="lazy">
+    <div class="page-header min-vh-25" style="background-image: url({{url_for('static', filename='/assets/img/EveAI_bg.jpg')}}); background-position: top left; background-repeat: no-repeat; background-size: cover;" loading="lazy">
         <span class="mask bg-gradient-primary opacity-4"></span>
         <div class="container">
             <div class="row">
@@ -10,4 +10,4 @@
             </div>
         </div>
     </div>
-</header>
+</header>

eveai_app/templates/interaction/view_chat_session.html

@@ -54,7 +54,7 @@
                             {% if embedding.url %}
                             <a href="{{ embedding.url }}" target="_blank">{{ embedding.url }}</a>
                             {% else %}
-                            {{ embedding.file_name }}
+                            {{ embedding.object_name }}
                             {% endif %}
                         </li>
                         {% endfor %}

eveai_app/templates/macros.html

@@ -1,16 +1,16 @@
-{% macro render_field(field, disabled_fields=[], exclude_fields=[]) %}
+{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %}
     {% set disabled = field.name in disabled_fields %}
     {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %}
     {% if field.name not in exclude_fields %}
         {% if field.type == 'BooleanField' %}
             <div class="form-check">
-                {{ field(class="form-check-input", type="checkbox", id="flexSwitchCheckDefault") }}
+                {{ field(class="form-check-input " + class, type="checkbox", id="flexSwitchCheckDefault") }}
                 {{ field.label(class="form-check-label", for="flexSwitchCheckDefault", disabled=disabled) }}
             </div>
         {% else %}
             <div class="form-group">
                 {{ field.label(class="form-label") }}
-                {{ field(class="form-control", disabled=disabled) }}
+                {{ field(class="form-control " + class, disabled=disabled) }}
                 {% if field.errors %}
                     <div class="invalid-feedback">
                         {% for error in field.errors %}
@@ -177,6 +177,48 @@
 </div>
 {% endmacro %}
 
+{% macro render_selectable_sortable_table_with_dict_headers(headers, rows, selectable, id, sort_by, sort_order) %}
+<div class="card">
+  <div class="table-responsive">
+    <table class="table align-items-center mb-0" id="{{ id }}">
+      <thead>
+        <tr>
+          {% if selectable %}
+            <th class="text-uppercase text-secondary text-xxs font-weight-bolder opacity-7">Select</th>
+          {% endif %}
+          {% for header in headers %}
+            <th class="text-uppercase text-secondary text-xxs font-weight-bolder opacity-7 sortable" data-sort="{{ header['sort'] }}">
+              {{ header['text'] }}
+              {% if sort_by == header['sort'] %}
+                {% if sort_order == 'asc' %}
+                  <i class="fas fa-sort-up"></i>
+                {% elif sort_order == 'desc' %}
+                  <i class="fas fa-sort-down"></i>
+                {% endif %}
+              {% else %}
+                <i class="fas fa-sort"></i>
+              {% endif %}
+            </th>
+          {% endfor %}
+        </tr>
+      </thead>
+      <tbody>
+        {% for row in rows %}
+          <tr>
+            {% if selectable %}
+              <td><input type="radio" name="selected_row" value="{{ row[0].value }}"></td>
+            {% endif %}
+            {% for cell in row %}
+              <td>{{ cell.value }}</td>
+            {% endfor %}
+          </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+{% endmacro %}
+
 {% macro render_accordion(accordion_id, accordion_items, header_title, header_description) %}
 <div class="accordion-1">
   <div class="container">

eveai_app/templates/navbar.html

@@ -81,6 +81,8 @@
                             {% endif %}
                             {% if current_user.is_authenticated %}
                                 {{ dropdown('Document Mgmt', 'note_stack', [
+                                    {'name': 'Add Catalog', 'url': '/document/catalog', 'roles': ['Super User', 'Tenant Admin']},
+                                    {'name': 'All Catalogs', 'url': '/document/catalogs', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Add URL', 'url': '/document/add_url', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Add a list of URLs', 'url': '/document/add_urls', 'roles': ['Super User', 'Tenant Admin']},
@@ -94,6 +96,14 @@
                                     {'name': 'Chat Sessions', 'url': '/interaction/chat_sessions', 'roles': ['Super User', 'Tenant Admin']},
                                 ]) }}
                             {% endif %}
+                            {% if current_user.is_authenticated %}
+                                {{ dropdown('Administration', 'settings', [
+                                    {'name': 'License Tier Registration', 'url': '/entitlements/license_tier', 'roles': ['Super User']},
+                                    {'name': 'All License Tiers', 'url': '/entitlements/view_license_tiers', 'roles': ['Super User']},
+                                    {'name': 'Trigger Actions', 'url': '/administration/trigger_actions', 'roles': ['Super User']},
+                                    {'name': 'Usage', 'url': '/entitlements/view_usages', 'roles': ['Super User', 'Tenant Admin']},
+                                ]) }}
+                            {% endif %}
                             {% if current_user.is_authenticated %}
                                 {{ dropdown(current_user.user_name, 'person', [
                                     {'name': 'Session Defaults', 'url': '/session_defaults', 'roles': ['Super User', 'Tenant Admin']},
@@ -106,6 +116,17 @@
                             {% endif %}
                         </ul>
                         {% if current_user.is_authenticated %}
+                            <ul class="navbar-nav d-lg-block d-none">
+                                <li class="nav-item">
+                                    <a href="/document/catalogs" class="btn btn-sm  bg-gradient-primary  mb-0 me-2">
+                                        {% if 'catalog_name' in session %}
+                                            CATALOG: {{ session['catalog_name'] }}
+                                        {% else %}
+                                            CHOOSE CATALOG
+                                        {% endif %}
+                                    </a>
+                                </li>
+                            </ul>
                             <ul class="navbar-nav d-lg-block d-none">
                                 <li class="nav-item">
                                     <a href="/session_defaults" class="btn btn-sm  bg-gradient-primary  mb-0">

eveai_app/templates/scripts.html

@@ -13,3 +13,5 @@
         <script src="{{url_for('static', filename='assets/js/plugins/anime.min.js')}}"></script>
         <script src="{{url_for('static', filename='assets/js/material-kit-pro.min.js')}}?v=3.0.4 type="text/javascript"></script>
         <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/js/bootstrap.bundle.min.js"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.13/js/select2.min.js"></script>
+

eveai_app/templates/user/select_tenant.html

@@ -1,22 +1,52 @@
 {% extends 'base.html' %}
-{% from "macros.html" import render_selectable_table, render_pagination %}
+{% from "macros.html" import render_selectable_table, render_pagination, render_field %}
-
 {% block title %}Tenant Selection{% endblock %}
-
 {% block content_title %}Select a Tenant{% endblock %}
 {% block content_description %}Select the active tenant for the current session{% endblock %}
-
 {% block content %}
+
+<!-- Filter Form -->
+<form method="POST" action="{{ url_for('user_bp.select_tenant') }}" class="mb-4">
+    {{ filter_form.hidden_tag() }}
+    <div class="row">
+        <div class="col-md-4">
+            {{ render_field(filter_form.types, class="select2") }}
+        </div>
+        <div class="col-md-4">
+            {{ render_field(filter_form.search) }}
+        </div>
+        <div class="col-md-4">
+            {{ filter_form.submit(class="btn btn-primary") }}
+        </div>
+    </div>
+</form>
+
+<!-- Tenant Selection Form -->
 <form method="POST" action="{{ url_for('user_bp.handle_tenant_selection') }}">
-    {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website"], rows=rows, selectable=True, id="tenantsTable") }}
+    {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website", "Type"], rows=rows, selectable=True, id="tenantsTable") }}
     <div class="form-group mt-3">
         <button type="submit" name="action" value="select_tenant" class="btn btn-primary">Set Session Tenant</button>
         <button type="submit" name="action" value="edit_tenant" class="btn btn-secondary">Edit Tenant</button>
     </div>
 </form>
+
 {% endblock %}
 
 {% block content_footer %}
-    {{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{% endblock %}
+
+{% block scripts %}
+<script>
+$(document).ready(function() {
+    $('.select2').select2({
+        placeholder: "Select tenant types",
+        allowClear: true,
+        minimumResultsForSearch: Infinity, // Hides the search box
+        dropdownCssClass: 'select2-dropdown-hidden', // Custom class for dropdown
+        containerCssClass: 'select2-container-hidden' // Custom class for container
+    });
+});
+</script>
 {% endblock %}
 

eveai_app/templates/user/tenant.html

@@ -1,21 +1,185 @@
 {% extends 'base.html' %}
-{% from "macros.html" import render_field %}
+{% from "macros.html" import render_field, render_included_field %}
 
-{% block title %}Tenant Registration{% endblock %}
+{% block title %}Create or Edit Tenant{% endblock %}
 
-{% block content_title %}Register Tenant{% endblock %}
+{% block content_title %}Create or Edit Tenant{% endblock %}
-{% block content_description %}Add a new tenant to EveAI{% endblock %}
+{% block content_description %}Create or Edit Tenant{% endblock %}
 
 {% block content %}
     <form method="post">
         {{ form.hidden_tag() }}
-        {%  set disabled_fields = [] %}
+        <!-- Main Tenant Information -->
-        {%  set exclude_fields = [] %}
+        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages', 'timezone','rag_context', 'type'] %}
         {% for field in form %}
-            {{ render_field(field, disabled_fields, exclude_fields) }}
+            {{ render_included_field(field, disabled_fields=[], include_fields=main_fields) }}
         {% endfor %}
-        <button type="submit" class="btn btn-primary">Register Tenant</button>
+
+        <!-- Nav Tabs -->
+        <div class="row mt-5">
+            <div class="col-lg-12">
+                <div class="nav-wrapper position-relative end-0">
+                    <ul class="nav nav-pills nav-fill p-1" role="tablist">
+                        <li class="nav-item" role="presentation">
+                            <a class="nav-link mb-0 px-0 py-1 active" data-toggle="tab" href="#model-info-tab" role="tab" aria-controls="model-info" aria-selected="true">
+                                Model Information
+                            </a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#license-info-tab" role="tab" aria-controls="license-info" aria-selected="false">
+                                License Information
+                            </a>
+                        </li>
+                    </ul>
+                </div>
+                <div class="tab-content tab-space">
+                    <!-- Model Information Tab -->
+                    <div class="tab-pane fade show active" id="model-info-tab" role="tabpanel">
+                        {% set model_fields = ['embedding_model', 'llm_model'] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=model_fields) }}
+                        {% endfor %}
+                    </div>
+                    <!-- License Information Tab -->
+                    <div class="tab-pane fade" id="license-info-tab" role="tabpanel">
+                        {% set license_fields = ['currency', 'usage_email', ] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=[], include_fields=license_fields) }}
+                        {% endfor %}
+                        <!-- Register API Key Button -->
+                        <div class="form-group">
+                            <button type="button" class="btn btn-primary" onclick="generateNewChatApiKey()">Register Chat API Key</button>
+                            <button type="button" class="btn btn-primary" onclick="generateNewApiKey()">Register API Key</button>
+                        </div>
+                        <!-- API Key Display Field -->
+                        <div id="chat-api-key-field" style="display:none;">
+                            <label for="chat-api-key">Chat API Key:</label>
+                            <input type="text" id="chat-api-key" class="form-control" readonly>
+                            <button type="button" id="copy-chat-button" class="btn btn-primary">Copy to Clipboard</button>
+                            <p id="copy-chat-message" style="display:none;color:green;">Chat API key copied to clipboard</p>
+                        </div>
+                        <div id="api-key-field" style="display:none;">
+                            <label for="api-key">API Key:</label>
+                            <input type="text" id="api-key" class="form-control" readonly>
+                            <button type="button" id="copy-api-button" class="btn btn-primary">Copy to Clipboard</button>
+                            <p id="copy-message" style="display:none;color:green;">API key copied to clipboard</p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <button type="submit" class="btn btn-primary">Save Tenant</button>
     </form>
 {% endblock %}
 
-{% block content_footer %} {% endblock %}
+
+{% block content_footer %}
+
+{% endblock %}
+
+{% block scripts %}
+<script>
+    // Function to generate a new Chat API Key
+    function generateNewChatApiKey() {
+        generateApiKey('/admin/user/generate_chat_api_key', '#chat-api-key', '#chat-api-key-field');
+    }
+
+    // Function to generate a new general API Key
+    function generateNewApiKey() {
+        generateApiKey('/admin/user/generate_api_api_key', '#api-key', '#api-key-field');
+    }
+
+    // Reusable function to handle API key generation
+    function generateApiKey(url, inputSelector, fieldSelector) {
+        $.ajax({
+            url: url,
+            type: 'POST',
+            contentType: 'application/json',
+            success: function(response) {
+                $(inputSelector).val(response.api_key);
+                $(fieldSelector).show();
+            },
+            error: function(error) {
+                alert('Error generating new API key: ' + error.responseText);
+            }
+        });
+    }
+
+    // Function to copy text to clipboard
+    function copyToClipboard(selector, messageSelector) {
+        const element = document.querySelector(selector);
+        if (element) {
+            const text = element.value;
+            if (navigator.clipboard && navigator.clipboard.writeText) {
+                navigator.clipboard.writeText(text).then(function() {
+                    showCopyMessage(messageSelector);
+                }).catch(function(error) {
+                    alert('Failed to copy text: ' + error);
+                });
+            } else {
+                fallbackCopyToClipboard(text, messageSelector);
+            }
+        } else {
+            console.error('Element not found for selector:', selector);
+        }
+    }
+
+    // Fallback method for copying text to clipboard
+    function fallbackCopyToClipboard(text, messageSelector) {
+        const textArea = document.createElement('textarea');
+        textArea.value = text;
+        document.body.appendChild(textArea);
+        textArea.focus();
+        textArea.select();
+        try {
+            document.execCommand('copy');
+            showCopyMessage(messageSelector);
+        } catch (err) {
+            alert('Fallback: Oops, unable to copy', err);
+        }
+        document.body.removeChild(textArea);
+    }
+
+    // Function to show copy confirmation message
+    function showCopyMessage(messageSelector) {
+        const message = document.querySelector(messageSelector);
+        if (message) {
+            message.style.display = 'block';
+            setTimeout(function() {
+                message.style.display = 'none';
+            }, 2000);
+        }
+    }
+
+    // Event listeners for copy buttons
+    document.getElementById('copy-chat-button').addEventListener('click', function() {
+        copyToClipboard('#chat-api-key', '#copy-chat-message');
+    });
+
+    document.getElementById('copy-api-button').addEventListener('click', function() {
+        copyToClipboard('#api-key', '#copy-message');
+    });
+</script>
+<script>
+    // JavaScript to detect user's timezone
+    document.addEventListener('DOMContentLoaded', (event) => {
+        // Detect timezone
+        const userTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+        // Send timezone to the server via a POST request
+        fetch('/set_user_timezone', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ timezone: userTimezone })
+        }).then(response => {
+            if (response.ok) {
+                console.log('Timezone sent to server successfully');
+            } else {
+                console.error('Failed to send timezone to server');
+            }
+        });
+    });
+</script>
+{% endblock %}

eveai_app/templates/user/tenant_overview.html

@@ -10,13 +10,13 @@
     <form method="post">
         {{ form.hidden_tag() }}
         <!-- Main Tenant Information -->
-        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages'] %}
+        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages', 'rag_context', 'type'] %}
         {% for field in form %}
             {{ render_included_field(field, disabled_fields=main_fields, include_fields=main_fields) }}
         {% endfor %}
 
         <!-- Nav Tabs -->
-        <div class="row">
+        <div class="row mt-5">
             <div class="col-lg-12">
                 <div class="nav-wrapper position-relative end-0">
                     <ul class="nav nav-pills nav-fill p-1" role="tablist">
@@ -30,21 +30,6 @@
                                 License Information
                             </a>
                         </li>
-                        <li class="nav-item">
-                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#chunking-tab" role="tab" aria-controls="chunking" aria-selected="false">
-                                Chunking
-                            </a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#embedding-search-tab" role="tab" aria-controls="html-chunking" aria-selected="false">
-                                Embedding Search
-                            </a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#tuning-tab" role="tab" aria-controls="html-chunking" aria-selected="false">
-                                Tuning
-                            </a>
-                        </li>
                     </ul>
                 </div>
                 <div class="tab-content tab-space">
@@ -57,7 +42,7 @@
                     </div>
                     <!-- License Information Tab -->
                     <div class="tab-pane fade" id="license-info-tab" role="tabpanel">
-                        {% set license_fields = ['license_start_date', 'license_end_date', 'allowed_monthly_interactions', ] %}
+                        {% set license_fields = ['currency', 'usage_email', ] %}
                         {% for field in form %}
                             {{ render_included_field(field, disabled_fields=license_fields, include_fields=license_fields) }}
                         {% endfor %}
@@ -78,27 +63,6 @@
                             <p id="copy-message" style="display:none;color:green;">API key copied to clipboard</p>
                         </div>
                     </div>
-                    <!-- Chunking Settings Tab -->
-                    <div class="tab-pane fade" id="chunking-tab" role="tabpanel">
-                        {% set html_fields = ['html_tags', 'html_end_tags', 'html_included_elements', 'html_excluded_elements', 'html_excluded_classes', 'min_chunk_size', 'max_chunk_size'] %}
-                        {% for field in form %}
-                            {{ render_included_field(field, disabled_fields=html_fields, include_fields=html_fields) }}
-                        {% endfor %}
-                    </div>
-                    <!-- Embedding Search Settings Tab -->
-                    <div class="tab-pane fade" id="embedding-search-tab" role="tabpanel">
-                        {% set es_fields = ['es_k', 'es_similarity_threshold', ] %}
-                        {% for field in form %}
-                            {{ render_included_field(field, disabled_fields=es_fields, include_fields=es_fields) }}
-                        {% endfor %}
-                    </div>
-                    <!-- Tuning Settings Tab -->
-                    <div class="tab-pane fade" id="tuning-tab" role="tabpanel">
-                        {% set tuning_fields = ['embed_tuning', 'rag_tuning', ] %}
-                        {% for field in form %}
-                            {{ render_included_field(field, disabled_fields=tuning_fields, include_fields=tuning_fields) }}
-                        {% endfor %}
-                    </div>
                 </div>
             </div>
         </div>

eveai_app/views/administration_forms.py

@@ -0,0 +1,7 @@
+from flask import current_app
+from flask_wtf import FlaskForm
+from wtforms.fields.simple import SubmitField
+
+
+class TriggerActionForm(FlaskForm):
+    submit = SubmitField('Submit')

eveai_app/views/administration_views.py

@@ -0,0 +1,39 @@
+import uuid
+from datetime import datetime as dt, timezone as tz
+from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify
+from flask_security import hash_password, roles_required, roles_accepted, current_user
+from itsdangerous import URLSafeTimedSerializer
+from sqlalchemy.exc import SQLAlchemyError
+
+from common.utils.celery_utils import current_celery
+from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
+from common.utils.nginx_utils import prefixed_url_for
+from .administration_forms import TriggerActionForm
+
+administration_bp = Blueprint('administration_bp', __name__, url_prefix='/administration')
+
+
+@administration_bp.route('/trigger_actions', methods=['GET'])
+@roles_accepted('Super User')
+def trigger_actions():
+    form = TriggerActionForm()
+    return render_template('administration/trigger_actions.html', form=form)
+
+
+@administration_bp.route('/handle_trigger_action', methods=['POST'])
+@roles_accepted('Super User')
+def handle_trigger_action():
+    action = request.form['action']
+    match action:
+        case 'update_usages':
+            try:
+                # Use send_task to trigger the task since it's part of another component (eveai_entitlements)
+                task = current_celery.send_task('update_usages', queue='entitlements')
+
+                current_app.logger.info(f"Usage update task triggered: {task.id}")
+                flash('Usage update task has been triggered successfully!', 'success')
+            except Exception as e:
+                current_app.logger.error(f"Failed to trigger usage update task: {str(e)}")
+                flash(f'Failed to trigger usage update: {str(e)}', 'danger')
+
+    return redirect(prefixed_url_for('administration_bp.trigger_actions'))

eveai_app/views/document_forms.py

@@ -1,8 +1,8 @@
 from flask import session, current_app
 from flask_wtf import FlaskForm
-from wtforms import (StringField, BooleanField, SubmitField, DateField,
+from wtforms import (StringField, BooleanField, SubmitField, DateField, IntegerField, FloatField, SelectMultipleField,
                      SelectField, FieldList, FormField, TextAreaField, URLField)
-from wtforms.validators import DataRequired, Length, Optional, URL, ValidationError
+from wtforms.validators import DataRequired, Length, Optional, URL, ValidationError, NumberRange
 from flask_wtf.file import FileField, FileAllowed, FileRequired
 import json
 
@@ -23,6 +23,36 @@ def validate_json(form, field):
             raise ValidationError('Invalid JSON format')
 
 
+class CatalogForm(FlaskForm):
+    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
+    description = TextAreaField('Description', validators=[Optional()])
+    # HTML Embedding Variables
+    html_tags = StringField('HTML Tags', validators=[DataRequired()],
+                            default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
+    html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
+                                default='p, li')
+    html_included_elements = StringField('HTML Included Elements', validators=[Optional()])
+    html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()])
+    html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
+    min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
+                                  default=2000)
+    max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
+                                  default=3000)
+    # Embedding Search variables
+    es_k = IntegerField('Limit for Searching Embeddings (5)',
+                        default=5,
+                        validators=[NumberRange(min=0)])
+    es_similarity_threshold = FloatField('Similarity Threshold for Searching Embeddings (0.5)',
+                                         default=0.5,
+                                         validators=[NumberRange(min=0, max=1)])
+    # Chat Variables
+    chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)])
+    chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)])
+    # Tuning variables
+    embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
+    rag_tuning = BooleanField('Enable RAG Tuning', default=False)
+
+
 class AddDocumentForm(FlaskForm):
     file = FileField('File', validators=[FileRequired(), allowed_file])
     name = StringField('Name', validators=[Length(max=100)])
@@ -30,7 +60,6 @@ class AddDocumentForm(FlaskForm):
     user_context = TextAreaField('User Context', validators=[Optional()])
     valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
     user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
-    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
 
     submit = SubmitField('Submit')
 
@@ -38,7 +67,8 @@ class AddDocumentForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
-        self.language.data = session.get('tenant').get('default_language')
+        if not self.language.data:
+            self.language.data = session.get('tenant').get('default_language')
 
 
 class AddURLForm(FlaskForm):
@@ -48,7 +78,6 @@ class AddURLForm(FlaskForm):
     user_context = TextAreaField('User Context', validators=[Optional()])
     valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
     user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
-    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
 
     submit = SubmitField('Submit')
 
@@ -56,7 +85,8 @@ class AddURLForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
-        self.language.data = session.get('tenant').get('default_language')
+        if not self.language.data:
+            self.language.data = session.get('tenant').get('default_language')
 
 
 class AddURLsForm(FlaskForm):
@@ -72,7 +102,8 @@ class AddURLsForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
-        self.language.data = session.get('tenant').get('default_language')
+        if not self.language.data:
+            self.language.data = session.get('tenant').get('default_language')
 
 
 class EditDocumentForm(FlaskForm):

eveai_app/views/document_list_view.py

@@ -0,0 +1,102 @@
+from datetime import datetime
+from flask import request, render_template, session
+from sqlalchemy import desc, asc, or_, and_, cast, Integer
+from common.models.document import Document, Catalog
+from common.utils.filtered_list_view import FilteredListView
+from common.utils.view_assistants import prepare_table_for_macro
+
+
+class DocumentListView(FilteredListView):
+    allowed_filters = ['catalog_id', 'validity']
+    allowed_sorts = ['id', 'name', 'catalog_name', 'valid_from', 'valid_to']
+
+    def get_query(self):
+        return Document.query.join(Catalog).add_columns(
+            Document.id,
+            Document.name,
+            Catalog.name.label('catalog_name'),
+            Document.valid_from,
+            Document.valid_to
+        )
+
+    def apply_filters(self, query):
+        filters = request.args.to_dict(flat=False)
+
+        if 'catalog_id' in filters:
+            catalog_ids = filters['catalog_id']
+            if catalog_ids:
+                # Convert catalog_ids to a list of integers
+                catalog_ids = [int(cid) for cid in catalog_ids if cid.isdigit()]
+                if catalog_ids:
+                    query = query.filter(Document.catalog_id.in_(catalog_ids))
+
+        if 'validity' in filters:
+            now = datetime.utcnow().date()
+            if 'valid' in filters['validity']:
+                query = query.filter(
+                    and_(
+                        or_(Document.valid_from.is_(None), Document.valid_from <= now),
+                        or_(Document.valid_to.is_(None), Document.valid_to >= now)
+                    )
+                )
+
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by', 'id')
+        sort_order = request.args.get('sort_order', 'asc')
+
+        if sort_by in self.allowed_sorts:
+            if sort_by == 'catalog_name':
+                column = Catalog.name
+            else:
+                column = getattr(Document, sort_by)
+
+            if sort_order == 'asc':
+                query = query.order_by(asc(column))
+            elif sort_order == 'desc':
+                query = query.order_by(desc(column))
+
+        return query
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        def format_date(date):
+            if isinstance(date, datetime):
+                return date.strftime('%Y-%m-%d')
+            elif isinstance(date, str):
+                return date
+            else:
+                return ''
+
+        rows = [
+            [
+                {'value': item.id, 'class': '', 'type': 'text'},
+                {'value': item.name, 'class': '', 'type': 'text'},
+                {'value': item.catalog_name, 'class': '', 'type': 'text'},
+                {'value': format_date(item.valid_from), 'class': '', 'type': 'text'},
+                {'value': format_date(item.valid_to), 'class': '', 'type': 'text'}
+            ] for item in pagination.items
+        ]
+
+        catalogs = Catalog.query.all()
+
+        context = {
+            'rows': rows,
+            'pagination': pagination,
+            'filters': request.args.to_dict(flat=False),
+            'sort_by': request.args.get('sort_by', 'id'),
+            'sort_order': request.args.get('sort_order', 'asc'),
+            'filter_options': self.get_filter_options(catalogs)
+        }
+        return render_template(self.template, **context)
+
+    def get_filter_options(self, catalogs):
+        return {
+            'catalog_id': [(str(cat.id), cat.name) for cat in catalogs],
+            'validity': [('valid', 'Valid'), ('all', 'All')]
+        }

eveai_app/views/document_version_list_view.py

@@ -12,7 +12,7 @@ class DocumentVersionListView(FilteredListView):
     allowed_sorts = ['id', 'processing_started_at', 'processing_finished_at', 'processing_error']
 
     def get_query(self):
-        return DocumentVersion.query.join(Document).filter(Document.tenant_id == session.get('tenant', {}).get('id'))
+        return DocumentVersion.query.join(Document)
 
     def apply_filters(self, query):
         filters = request.args.to_dict()

eveai_app/views/document_views.py

@@ -1,9 +1,11 @@
 import ast
 from datetime import datetime as dt, timezone as tz
 
+from babel.messages.setuptools_frontend import update_catalog
 from flask import request, redirect, flash, render_template, Blueprint, session, current_app
 from flask_security import roles_accepted, current_user
 from sqlalchemy import desc
+from sqlalchemy.orm import aliased
 from werkzeug.utils import secure_filename
 from sqlalchemy.exc import SQLAlchemyError
 import requests
@@ -12,18 +14,20 @@ from urllib.parse import urlparse, unquote
 import io
 import json
 
-from common.models.document import Document, DocumentVersion
+from common.models.document import Document, DocumentVersion, Catalog
 from common.extensions import db, minio_client
 from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
     process_multiple_urls, get_documents_list, edit_document, \
     edit_document_version, refresh_document
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException
-from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm
+from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
+    CatalogForm
 from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
 from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro, form_to_dict
+from .document_list_view import DocumentListView
 from .document_version_list_view import DocumentVersionListView
 
 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -52,31 +56,148 @@ def before_request():
         raise
 
 
+@document_bp.route('/catalog', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def catalog():
+    form = CatalogForm()
+
+    if form.validate_on_submit():
+        tenant_id = session.get('tenant').get('id')
+        new_catalog = Catalog()
+        form.populate_obj(new_catalog)
+        # Handle Embedding Variables
+        new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
+        new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
+            if form.html_end_tags.data else []
+        new_catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
+            if form.html_included_elements.data else []
+        new_catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
+            if form.html_excluded_elements.data else []
+        new_catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
+            if form.html_excluded_classes.data else []
+        set_logging_information(new_catalog, dt.now(tz.utc))
+
+        try:
+            db.session.add(new_catalog)
+            db.session.commit()
+            flash('Catalog successfully added!', 'success')
+            current_app.logger.info(f'Catalog {new_catalog.name} successfully added for tenant {tenant_id}!')
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash(f'Failed to add catalog. Error: {e}', 'danger')
+            current_app.logger.error(f'Failed to add catalog {new_catalog.name}'
+                                     f'for tenant {tenant_id}. Error: {str(e)}')
+
+    return render_template('document/catalog.html', form=form)
+
+
+@document_bp.route('/catalogs', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def catalogs():
+    page = request.args.get('page', 1, type=int)
+    per_page = request.args.get('per_page', 10, type=int)
+
+    query = Catalog.query.order_by(Catalog.id)
+
+    pagination = query.paginate(page=page, per_page=per_page)
+    the_catalogs = pagination.items
+
+    # prepare table data
+    rows = prepare_table_for_macro(the_catalogs, [('id', ''), ('name', '')])
+
+    # Render the catalogs in a template
+    return render_template('document/catalogs.html', rows=rows, pagination=pagination)
+
+
+@document_bp.route('/handle_catalog_selection', methods=['POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def handle_catalog_selection():
+    catalog_identification = request.form.get('selected_row')
+    catalog_id = ast.literal_eval(catalog_identification).get('value')
+    action = request.form['action']
+    catalog = Catalog.query.get_or_404(catalog_id)
+
+    if action == 'set_session_catalog':
+        current_app.logger.info(f'Setting session catalog to {catalog.name}')
+        session['catalog_id'] = catalog_id
+        session['catalog_name'] = catalog.name
+        current_app.logger.info(f'Finished setting session catalog to {catalog.name}')
+    elif action == 'edit_catalog':
+        return redirect(prefixed_url_for('document_bp.edit_catalog', catalog_id=catalog_id))
+
+    return redirect(prefixed_url_for('document_bp.catalogs'))
+
+
+@document_bp.route('/catalog/<int:catalog_id>', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def edit_catalog(catalog_id):
+    catalog = Catalog.query.get_or_404(catalog_id)
+    form = CatalogForm(obj=catalog)
+    tenant_id = session.get('tenant').get('id')
+
+    # Convert arrays to comma-separated strings for display
+    if request.method == 'GET':
+        form.html_tags.data = ', '.join(catalog.html_tags or '')
+        form.html_end_tags.data = ', '.join(catalog.html_end_tags or '')
+        form.html_included_elements.data = ', '.join(catalog.html_included_elements or '')
+        form.html_excluded_elements.data = ', '.join(catalog.html_excluded_elements or '')
+        form.html_excluded_classes.data = ', '.join(catalog.html_excluded_classes or '')
+
+    if request.method == 'POST' and form.validate_on_submit():
+        form.populate_obj(catalog)
+        # Handle Embedding Variables
+        catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
+        catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
+            if form.html_end_tags.data else []
+        catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
+            if form.html_included_elements.data else []
+        catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
+            if form.html_excluded_elements.data else []
+        catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
+            if form.html_excluded_classes.data else []
+        update_logging_information(catalog, dt.now(tz.utc))
+        try:
+            db.session.add(catalog)
+            db.session.commit()
+            flash('Catalog successfully updated successfully!', 'success')
+            current_app.logger.info(f'Catalog {catalog.name} successfully updated for tenant {tenant_id}')
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash(f'Failed to update catalog. Error: {e}', 'danger')
+            current_app.logger.error(f'Failed to update catalog {catalog_id} for tenant {tenant_id}. Error: {str(e)}')
+
+        return redirect(prefixed_url_for('document_bp.catalogs'))
+    else:
+        form_validation_failed(request, form)
+
+    return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id)
+
+
 @document_bp.route('/add_document', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
 def add_document():
     form = AddDocumentForm()
-    current_app.logger.debug('Adding document')
 
     if form.validate_on_submit():
         try:
-            current_app.logger.debug('Validating file type')
             tenant_id = session['tenant']['id']
+            catalog_id = session['catalog_id']
             file = form.file.data
             filename = secure_filename(file.filename)
             extension = filename.rsplit('.', 1)[1].lower()
 
             validate_file_type(extension)
 
+            current_app.logger.debug(f'Language on form: {form.language.data}')
             api_input = {
+                'catalog_id': catalog_id,
                 'name': form.name.data,
                 'language': form.language.data,
                 'user_context': form.user_context.data,
                 'valid_from': form.valid_from.data,
                 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
-                'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None
-
             }
+            current_app.logger.debug(f'Creating document stack with input {api_input}')
 
             new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
             task_id = start_embedding_task(tenant_id, new_doc_vers.id)
@@ -102,18 +223,19 @@ def add_url():
     if form.validate_on_submit():
         try:
             tenant_id = session['tenant']['id']
+            catalog_id = session['catalog_id']
             url = form.url.data
 
             file_content, filename, extension = process_url(url, tenant_id)
 
             api_input = {
+                'catalog_id': catalog_id,
                 'name': form.name.data or filename,
                 'url': url,
                 'language': form.language.data,
                 'user_context': form.user_context.data,
                 'valid_from': form.valid_from.data,
                 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
-                'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None
             }
 
             new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id)
@@ -174,22 +296,23 @@ def add_urls():
 @document_bp.route('/documents', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
 def documents():
-    page = request.args.get('page', 1, type=int)
+    view = DocumentListView(Document, 'document/documents.html', per_page=10)
-    per_page = request.args.get('per_page', 10, type=int)
+    return view.get()
-
-    pagination = get_documents_list(page, per_page)
-    docs = pagination.items
-
-    rows = prepare_table_for_macro(docs, [('id', ''), ('name', ''), ('valid_from', ''), ('valid_to', '')])
-
-    return render_template('document/documents.html', rows=rows, pagination=pagination)
 
 
 @document_bp.route('/handle_document_selection', methods=['POST'])
 @roles_accepted('Super User', 'Tenant Admin')
 def handle_document_selection():
     document_identification = request.form['selected_row']
-    doc_id = ast.literal_eval(document_identification).get('value')
+    if isinstance(document_identification, int) or document_identification.isdigit():
+        doc_id = int(document_identification)
+    else:
+        # If it's not an integer, assume it's a string representation of a dictionary
+        try:
+            doc_id = ast.literal_eval(document_identification).get('value')
+        except (ValueError, AttributeError):
+            flash('Invalid document selection.', 'error')
+            return redirect(prefixed_url_for('document_bp.documents'))
 
     action = request.form['action']
 
@@ -211,9 +334,25 @@ def handle_document_selection():
 @document_bp.route('/edit_document/<int:document_id>', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
 def edit_document_view(document_id):
-    doc = Document.query.get_or_404(document_id)
+    # Use an alias for the Catalog to avoid column name conflicts
+    CatalogAlias = aliased(Catalog)
+
+    # Query for the document and its catalog
+    result = db.session.query(Document, CatalogAlias.name.label('catalog_name')) \
+        .join(CatalogAlias, Document.catalog_id == CatalogAlias.id) \
+        .filter(Document.id == document_id) \
+        .first_or_404()
+
+    doc, catalog_name = result
+
     form = EditDocumentForm(obj=doc)
 
+    if request.method == 'GET':
+        # Populate form with current values
+        form.name.data = doc.name
+        form.valid_from.data = doc.valid_from
+        form.valid_to.data = doc.valid_to
+
     if form.validate_on_submit():
         updated_doc, error = edit_document(
             document_id,
@@ -229,7 +368,7 @@ def edit_document_view(document_id):
     else:
         form_validation_failed(request, form)
 
-    return render_template('document/edit_document.html', form=form, document_id=document_id)
+    return render_template('document/edit_document.html', form=form, document_id=document_id, catalog_name=catalog_name)
 
 
 @document_bp.route('/edit_document_version/<int:document_version_id>', methods=['GET', 'POST'])
@@ -271,8 +410,8 @@ def document_versions(document_id):
     pagination = query.paginate(page=page, per_page=per_page, error_out=False)
     doc_langs = pagination.items
 
-    rows = prepare_table_for_macro(doc_langs, [('id', ''), ('url', ''), ('file_location', ''),
+    rows = prepare_table_for_macro(doc_langs, [('id', ''), ('url', ''),
-                                               ('file_name', ''), ('file_type', ''),
+                                               ('object_name', ''), ('file_type', ''),
                                                ('processing', ''), ('processing_started_at', ''),
                                                ('processing_finished_at', ''), ('processing_error', '')])
 
@@ -283,7 +422,15 @@ def document_versions(document_id):
 @roles_accepted('Super User', 'Tenant Admin')
 def handle_document_version_selection():
     document_version_identification = request.form['selected_row']
-    doc_vers_id = ast.literal_eval(document_version_identification).get('value')
+    if isinstance(document_version_identification, int) or document_version_identification.isdigit():
+        doc_vers_id = int(document_version_identification)
+    else:
+        # If it's not an integer, assume it's a string representation of a dictionary
+        try:
+            doc_vers_id = ast.literal_eval(document_version_identification).get('value')
+        except (ValueError, AttributeError):
+            flash('Invalid document version selection.', 'error')
+            return redirect(prefixed_url_for('document_bp.document_versions_list'))
 
     action = request.form['action']
 
@@ -335,7 +482,7 @@ def refresh_all_documents():
 
 
 def refresh_document_view(document_id):
-    new_version, result = refresh_document(document_id)
+    new_version, result = refresh_document(document_id, session['tenant']['id'])
     if new_version:
         flash(f'Document refreshed. New version: {new_version.id}. Task ID: {result}', 'success')
     else:
@@ -352,10 +499,9 @@ def re_embed_latest_versions():
 
 
 def process_version(version_id):
-    task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
+    task = current_celery.send_task('create_embeddings',
-        session['tenant']['id'],
+                                    args=[session['tenant']['id'], version_id,],
-        version_id,
+                                    queue='embeddings')
-    ])
     current_app.logger.info(f'Embedding creation retriggered by user {current_user.id}, {current_user.email} '
                             f'for tenant {session["tenant"]["id"]}, '
                             f'Document Version {version_id}. '
@@ -401,47 +547,47 @@ def fetch_html(url):
     return response.content
 
 
-def prepare_document_data(docs):
+# def prepare_document_data(docs):
-    rows = []
+#     rows = []
-    for doc in docs:
+#     for doc in docs:
-        doc_row = [{'value': doc.name, 'class': '', 'type': 'text'},
+#         doc_row = [{'value': doc.name, 'class': '', 'type': 'text'},
-                   {'value': doc.created_at.strftime("%Y-%m-%d %H:%M:%S"), 'class': '', 'type': 'text'}]
+#                    {'value': doc.created_at.strftime("%Y-%m-%d %H:%M:%S"), 'class': '', 'type': 'text'}]
-        # Document basic details
+#         # Document basic details
-        if doc.valid_from:
+#         if doc.valid_from:
-            doc_row.append({'value': doc.valid_from.strftime("%Y-%m-%d"), 'class': '', 'type': 'text'})
+#             doc_row.append({'value': doc.valid_from.strftime("%Y-%m-%d"), 'class': '', 'type': 'text'})
-        else:
+#         else:
-            doc_row.append({'value': '', 'class': '', 'type': 'text'})
+#             doc_row.append({'value': '', 'class': '', 'type': 'text'})
-
+#
-        # Nested languages and versions
+#         # Nested languages and versions
-        languages_rows = []
+#         languages_rows = []
-        for lang in doc.languages:
+#         for lang in doc.languages:
-            lang_row = [{'value': lang.language, 'class': '', 'type': 'text'}]
+#             lang_row = [{'value': lang.language, 'class': '', 'type': 'text'}]
-
+#
-            # Latest version details if available (should be available ;-) )
+#             # Latest version details if available (should be available ;-) )
-            if lang.latest_version:
+#             if lang.latest_version:
-                lang_row.append({'value': lang.latest_version.created_at.strftime("%Y-%m-%d %H:%M:%S"),
+#                 lang_row.append({'value': lang.latest_version.created_at.strftime("%Y-%m-%d %H:%M:%S"),
-                                 'class': '', 'type': 'text'})
+#                                  'class': '', 'type': 'text'})
-                if lang.latest_version.url:
+#                 if lang.latest_version.url:
-                    lang_row.append({'value': lang.latest_version.url,
+#                     lang_row.append({'value': lang.latest_version.url,
-                                     'class': '', 'type': 'link', 'href': lang.latest_version.url})
+#                                      'class': '', 'type': 'link', 'href': lang.latest_version.url})
-                else:
+#                 else:
-                    lang_row.append({'value': '', 'class': '', 'type': 'text'})
+#                     lang_row.append({'value': '', 'class': '', 'type': 'text'})
-
+#
-                if lang.latest_version.file_name:
+#                 if lang.latest_version.object_name:
-                    lang_row.append({'value': lang.latest_version.file_name, 'class': '', 'type': 'text'})
+#                     lang_row.append({'value': lang.latest_version.object_name, 'class': '', 'type': 'text'})
-                else:
+#                 else:
-                    lang_row.append({'value': '', 'class': '', 'type': 'text'})
+#                     lang_row.append({'value': '', 'class': '', 'type': 'text'})
-
+#
-                if lang.latest_version.file_type:
+#                 if lang.latest_version.file_type:
-                    lang_row.append({'value': lang.latest_version.file_type, 'class': '', 'type': 'text'})
+#                     lang_row.append({'value': lang.latest_version.file_type, 'class': '', 'type': 'text'})
-                else:
+#                 else:
-                    lang_row.append({'value': '', 'class': '', 'type': 'text'})
+#                     lang_row.append({'value': '', 'class': '', 'type': 'text'})
-                    # Include other details as necessary
+#                     # Include other details as necessary
-
+#
-            languages_rows.append(lang_row)
+#             languages_rows.append(lang_row)
-
+#
-        doc_row.append({'is_group': True, 'colspan': '5',
+#         doc_row.append({'is_group': True, 'colspan': '5',
-                        'headers': ['Language', 'Latest Version', 'URL', 'File Name', 'Type'],
+#                         'headers': ['Language', 'Latest Version', 'URL', 'File Name', 'Type'],
-                        'sub_rows': languages_rows})
+#                         'sub_rows': languages_rows})
-        rows.append(doc_row)
+#         rows.append(doc_row)
-    return rows
+#     return rows

eveai_app/views/entitlements_forms.py

@@ -0,0 +1,76 @@
+from flask import current_app
+from flask_wtf import FlaskForm
+from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
+                     SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField)
+from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError, InputRequired
+import pytz
+
+
+class LicenseTierForm(FlaskForm):
+    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
+    version = StringField('Version', validators=[DataRequired(), Length(max=50)])
+    start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
+    end_date = DateField('End Date', id='form-control datepicker', validators=[Optional()])
+    basic_fee_d = FloatField('Basic Fee ($)', validators=[InputRequired(), NumberRange(min=0)])
+    basic_fee_e = FloatField('Basic Fee (€)', validators=[InputRequired(), NumberRange(min=0)])
+    max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
+    additional_storage_price_d = FloatField('Additional Storage Fee ($)',
+                                            validators=[InputRequired(), NumberRange(min=0)])
+    additional_storage_price_e = FloatField('Additional Storage Fee (€)',
+                                            validators=[InputRequired(), NumberRange(min=0)])
+    additional_storage_bucket = IntegerField('Additional Storage Bucket Size (MiB)',
+                                             validators=[DataRequired(), NumberRange(min=1)])
+    included_embedding_mb = IntegerField('Included Embeddings (MiB)',
+                                             validators=[DataRequired(), NumberRange(min=1)])
+    additional_embedding_price_d = FloatField('Additional Embedding Fee ($)',
+                                              validators=[InputRequired(), NumberRange(min=0)])
+    additional_embedding_price_e = FloatField('Additional Embedding Fee (€)',
+                                              validators=[InputRequired(), NumberRange(min=0)])
+    additional_embedding_bucket = IntegerField('Additional Embedding Bucket Size (MiB)',
+                                               validators=[DataRequired(), NumberRange(min=1)])
+    included_interaction_tokens = IntegerField('Included Embedding Tokens',
+                                               validators=[DataRequired(), NumberRange(min=1)])
+    additional_interaction_token_price_d = FloatField('Additional Interaction Token Fee ($)',
+                                                      validators=[InputRequired(), NumberRange(min=0)])
+    additional_interaction_token_price_e = FloatField('Additional Interaction Token Fee (€)',
+                                                      validators=[InputRequired(), NumberRange(min=0)])
+    additional_interaction_bucket = IntegerField('Additional Interaction Bucket Size',
+                                                 validators=[DataRequired(), NumberRange(min=1)])
+    standard_overage_embedding = FloatField('Standard Overage Embedding (%)',
+                                            validators=[DataRequired(), NumberRange(min=0)],
+                                            default=0)
+    standard_overage_interaction = FloatField('Standard Overage Interaction (%)',
+                                              validators=[DataRequired(), NumberRange(min=0)],
+                                              default=0)
+
+
+class LicenseForm(FlaskForm):
+    start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
+    end_date = DateField('End Date', id='form-control datepicker', validators=[DataRequired()])
+    currency = StringField('Currency', validators=[Optional(), Length(max=20)])
+    yearly_payment = BooleanField('Yearly Payment', validators=[DataRequired()], default=False)
+    basic_fee = FloatField('Basic Fee', validators=[InputRequired(), NumberRange(min=0)])
+    max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
+    additional_storage_price = FloatField('Additional Storage Token Fee',
+                                          validators=[InputRequired(), NumberRange(min=0)])
+    additional_storage_bucket = IntegerField('Additional Storage Bucket Size (MiB)',
+                                             validators=[DataRequired(), NumberRange(min=1)])
+    included_embedding_mb = IntegerField('Included Embedding Tokens (MiB)',
+                                         validators=[DataRequired(), NumberRange(min=1)])
+    additional_embedding_price = FloatField('Additional Embedding Token Fee',
+                                            validators=[InputRequired(), NumberRange(min=0)])
+    additional_embedding_bucket = IntegerField('Additional Embedding Bucket Size (MiB)',
+                                               validators=[DataRequired(), NumberRange(min=1)])
+    included_interaction_tokens = IntegerField('Included Interaction Tokens',
+                                               validators=[DataRequired(), NumberRange(min=1)])
+    additional_interaction_token_price = FloatField('Additional Interaction Token Fee',
+                                                    validators=[InputRequired(), NumberRange(min=0)])
+    additional_interaction_bucket = IntegerField('Additional Interaction Bucket Size',
+                                                 validators=[DataRequired(), NumberRange(min=1)])
+    overage_embedding = FloatField('Overage Embedding (%)',
+                                   validators=[DataRequired(), NumberRange(min=0)],
+                                   default=0)
+    overage_interaction = FloatField('Overage Interaction (%)',
+                                     validators=[DataRequired(), NumberRange(min=0)],
+                                     default=0)
+

eveai_app/views/entitlements_views.py

@@ -0,0 +1,235 @@
+import uuid
+from datetime import datetime as dt, timezone as tz
+from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify
+from flask_security import hash_password, roles_required, roles_accepted, current_user
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import or_, desc
+import ast
+
+from common.models.entitlements import License, LicenseTier, LicenseUsage, BusinessEventLog
+from common.extensions import db, security, minio_client, simple_encryption
+from .entitlements_forms import LicenseTierForm, LicenseForm
+from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
+from common.utils.nginx_utils import prefixed_url_for
+
+entitlements_bp = Blueprint('entitlements_bp', __name__, url_prefix='/entitlements')
+
+
+@entitlements_bp.route('/license_tier', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def license_tier():
+    form = LicenseTierForm()
+    if form.validate_on_submit():
+        current_app.logger.info("Adding License Tier")
+
+        new_license_tier = LicenseTier()
+        form.populate_obj(new_license_tier)
+
+        try:
+            db.session.add(new_license_tier)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f'Failed to add license tier to database. Error: {str(e)}')
+            flash(f'Failed to add license tier to database. Error: {str(e)}', 'success')
+            return render_template('entitlements/license_tier.html', form=form)
+
+        current_app.logger.info(f"Successfully created license tier {new_license_tier.id}")
+        flash(f"Successfully created tenant license tier {new_license_tier.id}")
+
+        return redirect(prefixed_url_for('entitlements_bp.view_license_tiers'))
+    else:
+        form_validation_failed(request, form)
+
+    return render_template('entitlements/license_tier.html', form=form)
+
+
+@entitlements_bp.route('/view_license_tiers', methods=['GET', 'POST'])
+@roles_required('Super User')
+def view_license_tiers():
+    page = request.args.get('page', 1, type=int)
+    per_page = request.args.get('per_page', 10, type=int)
+    today = dt.now(tz.utc)
+
+    query = LicenseTier.query.filter(
+        or_(
+            LicenseTier.end_date == None,
+            LicenseTier.end_date >= today
+        )
+    ).order_by(LicenseTier.start_date.desc(), LicenseTier.id)
+
+    pagination = query.paginate(page=page, per_page=per_page, error_out=False)
+    license_tiers = pagination.items
+
+    rows = prepare_table_for_macro(license_tiers, [('id', ''), ('name', ''), ('version', ''), ('start_date', ''),
+                                                   ('end_date', '')])
+
+    return render_template('entitlements/view_license_tiers.html', rows=rows, pagination=pagination)
+
+
+@entitlements_bp.route('/handle_license_tier_selection', methods=['POST'])
+@roles_required('Super User')
+def handle_license_tier_selection():
+    license_tier_identification = request.form['selected_row']
+    license_tier_id = ast.literal_eval(license_tier_identification).get('value')
+    the_license_tier = LicenseTier.query.get(license_tier_id)
+
+    action = request.form['action']
+
+    match action:
+        case 'edit_license_tier':
+            return redirect(prefixed_url_for('entitlements_bp.edit_license_tier',
+                                             license_tier_id=license_tier_id))
+        case 'create_license_for_tenant':
+            return redirect(prefixed_url_for('entitlements_bp.create_license',
+                                             license_tier_id=license_tier_id))
+    # Add more conditions for other actions
+    return redirect(prefixed_url_for('entitlements_bp.view_license_tiers'))
+
+
+@entitlements_bp.route('/license_tier/<int:license_tier_id>', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def edit_license_tier(license_tier_id):
+    license_tier = LicenseTier.query.get_or_404(license_tier_id)  # This will return a 404 if no license tier is found
+    form = LicenseTierForm(obj=license_tier)
+
+    if form.validate_on_submit():
+        # Populate the license_tier with form data
+        form.populate_obj(license_tier)
+
+        try:
+            db.session.add(license_tier)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f'Failed to edit License Tier. Error: {str(e)}')
+            flash(f'Failed to edit License Tier. Error: {str(e)}', 'danger')
+            return render_template('entitlements/license_tier.html', form=form, license_tier_id=license_tier.id)
+
+        flash('License Tier updated successfully.', 'success')
+        return redirect(
+            prefixed_url_for('entitlements_bp.edit_license_tier', license_tier_id=license_tier_id))
+    else:
+        form_validation_failed(request, form)
+
+    return render_template('entitlements/license_tier.html', form=form, license_tier_id=license_tier.id)
+
+
+@entitlements_bp.route('/create_license/<int:license_tier_id>', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def create_license(license_tier_id):
+    form = LicenseForm()
+    tenant_id = session.get('tenant').get('id')
+    currency = session.get('tenant').get('currency')
+
+    if request.method == 'GET':
+        # Fetch the LicenseTier
+        license_tier = LicenseTier.query.get_or_404(license_tier_id)
+
+        # Prefill the form with LicenseTier data
+        # Currency depending data
+        if currency == '$':
+            form.basic_fee.data = license_tier.basic_fee_d
+            form.additional_storage_price.data = license_tier.additional_storage_price_d
+            form.additional_embedding_price.data = license_tier.additional_embedding_price_d
+            form.additional_interaction_token_price.data = license_tier.additional_interaction_token_price_d
+        elif currency == '€':
+            form.basic_fee.data = license_tier.basic_fee_e
+            form.additional_storage_price.data = license_tier.additional_storage_price_e
+            form.additional_embedding_price.data = license_tier.additional_embedding_price_e
+            form.additional_interaction_token_price.data = license_tier.additional_interaction_token_price_e
+        else:
+            current_app.logger.error(f'Invalid currency {currency} for tenant {tenant_id} while creating license.')
+            flash(f"Invalid currency {currency} for tenant {tenant_id} while creating license. "
+                  f"Check tenant's currency and try again.", 'danger')
+            return redirect(prefixed_url_for('user_bp.edit_tenant', tenant_id=tenant_id))
+        # General data
+        form.currency.data = currency
+        form.max_storage_mb.data = license_tier.max_storage_mb
+        form.additional_storage_bucket.data = license_tier.additional_storage_bucket
+        form.included_embedding_mb.data = license_tier.included_embedding_mb
+        form.additional_embedding_bucket.data = license_tier.additional_embedding_bucket
+        form.included_interaction_tokens.data = license_tier.included_interaction_tokens
+        form.additional_interaction_bucket.data = license_tier.additional_interaction_bucket
+        form.overage_embedding.data = license_tier.standard_overage_embedding
+        form.overage_interaction.data = license_tier.standard_overage_interaction
+    else:   # POST
+        # Create a new License instance
+        new_license = License(
+            tenant_id=tenant_id,
+            tier_id=license_tier_id,
+        )
+        current_app.logger.debug(f"Currency data in form: {form.currency.data}")
+        if form.validate_on_submit():
+            # Update the license with form data
+            form.populate_obj(new_license)
+            # Currency is added here again, as a form doesn't include disabled fields when passing it in the request
+            new_license.currency = currency
+
+            try:
+                db.session.add(new_license)
+                db.session.commit()
+                flash('License created successfully', 'success')
+                return redirect(prefixed_url_for('entitlements_bp.edit_license', license_id=new_license.id))
+            except Exception as e:
+                db.session.rollback()
+                flash(f'Error creating license: {str(e)}', 'error')
+        else:
+            form_validation_failed(request, form)
+
+    return render_template('entitlements/license.html', form=form, ext_disabled_fields=[])
+
+
+@entitlements_bp.route('/license/<int:license_id>', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def edit_license(license_id):
+    license = License.query.get_or_404(license_id)  # This will return a 404 if no license tier is found
+    form = LicenseForm(obj=license)
+    disabled_fields = []
+    if len(license.usages) > 0:     # There already are usage records linked to this license
+        # Define which fields should be disabled
+        disabled_fields = [field.name for field in form if field.name != 'end_date']
+
+    if form.validate_on_submit():
+        # Populate the license with form data
+        form.populate_obj(license)
+
+        try:
+            db.session.add(license)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f'Failed to edit License. Error: {str(e)}')
+            flash(f'Failed to edit License. Error: {str(e)}', 'danger')
+            return render_template('entitlements/license.html', form=form)
+
+        flash('License updated successfully.', 'success')
+        return redirect(
+            prefixed_url_for('entitlements_bp.edit_license', license_tier_id=license_id))
+    else:
+        form_validation_failed(request, form)
+
+    return render_template('entitlements/license.html', form=form, license_tier_id=license_tier.id,
+                           ext_disabled_fields=disabled_fields)
+
+
+@entitlements_bp.route('/view_usages')
+@roles_accepted('Super User', 'Tenant Admin')
+def view_usages():
+    page = request.args.get('page', 1, type=int)
+    per_page = request.args.get('per_page', 10, type=int)
+
+    tenant_id = session.get('tenant').get('id')
+    query = LicenseUsage.query.filter_by(tenant_id=tenant_id).order_by(desc(LicenseUsage.id))
+
+    pagination = query.paginate(page=page, per_page=per_page)
+    lus = pagination.items
+
+    # prepare table data
+
+    rows = prepare_table_for_macro(lus, [('id', ''), ('period_start_date', ''), ('period_end_date', ''),
+                                         ('storage_mb_used', ''), ('embedding_mb_used', ''),
+                                         ('interaction_total_tokens_used', '')])
+
+    # Render the users in a template
+    return render_template('entitlements/view_usages.html', rows=rows, pagination=pagination)

eveai_app/views/healthz_views.py

@@ -0,0 +1,100 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+import time
+
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+# Define Prometheus metrics
+api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint'])
+api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        "minio": check_minio(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('ping', queue='embeddings')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+def check_minio():
+    try:
+        # List buckets to check if MinIO is accessible
+        minio_client.list_buckets()
+        return True
+    except Exception as e:
+        current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
+
+
+# Custom metrics example
+@healthz_bp.before_app_request
+def before_request():
+    request.start_time = time.time()
+    api_request_counter.labels(
+        method=request.method, endpoint=request.endpoint
+    ).inc()
+
+
+@healthz_bp.after_app_request
+def after_request(response):
+    request_duration = time.time() - request.start_time
+    api_request_latency.observe(request_duration)
+    return response
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )

eveai_app/views/interaction_views.py

@@ -93,17 +93,17 @@ def view_chat_session(chat_session_id):
     # Fetch all related embeddings for the interactions in this session
     embedding_query = (db.session.query(InteractionEmbedding.interaction_id,
                                         DocumentVersion.url,
-                                        DocumentVersion.file_name)
+                                        DocumentVersion.object_name)
                        .join(Embedding, InteractionEmbedding.embedding_id == Embedding.id)
                        .join(DocumentVersion, Embedding.doc_vers_id == DocumentVersion.id)
                        .filter(InteractionEmbedding.interaction_id.in_([i.id for i in interactions])))
 
     # Create a dictionary to store embeddings for each interaction
     embeddings_dict = {}
-    for interaction_id, url, file_name in embedding_query:
+    for interaction_id, url, object_name in embedding_query:
         if interaction_id not in embeddings_dict:
             embeddings_dict[interaction_id] = []
-        embeddings_dict[interaction_id].append({'url': url, 'file_name': file_name})
+        embeddings_dict[interaction_id].append({'url': url, 'object_name': object_name})
 
     return render_template('interaction/view_chat_session.html',
                            chat_session=chat_session,

eveai_app/views/user_forms.py

@@ -2,7 +2,7 @@ from flask import current_app
 from flask_wtf import FlaskForm
 from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
                      SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField)
-from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional
+from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError
 import pytz
 
 from common.models.user import Role
@@ -14,17 +14,18 @@ class TenantForm(FlaskForm):
     # language fields
     default_language = SelectField('Default Language', choices=[], validators=[DataRequired()])
     allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[DataRequired()])
+    # invoicing fields
+    currency = SelectField('Currency', choices=[], validators=[DataRequired()])
+    usage_email = EmailField('Usage Email', validators=[DataRequired(), Email()])
     # Timezone
     timezone = SelectField('Timezone', choices=[], validators=[DataRequired()])
     # RAG context
     rag_context = TextAreaField('RAG Context', validators=[Optional()])
+    # Tenant Type
+    type = SelectField('Tenant Type', validators=[Optional()], default='Active')
     # LLM fields
     embedding_model = SelectField('Embedding Model', choices=[], validators=[DataRequired()])
     llm_model = SelectField('Large Language Model', choices=[], validators=[DataRequired()])
-    # license fields
-    license_start_date = DateField('License Start Date', id='form-control datepicker', validators=[Optional()])
-    license_end_date = DateField('License End Date', id='form-control datepicker', validators=[Optional()])
-    allowed_monthly_interactions = IntegerField('Allowed Monthly Interactions', validators=[NumberRange(min=0)])
     # Embedding variables
     html_tags = StringField('HTML Tags', validators=[DataRequired()],
                             default='p, h1, h2, h3, h4, h5, h6, li')
@@ -57,6 +58,8 @@ class TenantForm(FlaskForm):
         # initialise language fields
         self.default_language.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
         self.allowed_languages.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
+        # initialise currency field
+        self.currency.choices = [(curr, curr) for curr in current_app.config['SUPPORTED_CURRENCIES']]
         # initialise timezone
         self.timezone.choices = [(tz, tz) for tz in pytz.all_timezones]
         # initialise LLM fields
@@ -65,6 +68,7 @@ class TenantForm(FlaskForm):
         # Initialize fallback algorithms
         self.fallback_algorithms.choices = \
             [(algorithm, algorithm.lower()) for algorithm in current_app.config['FALLBACK_ALGORITHMS']]
+        self.type.choices = [(t, t) for t in current_app.config['TENANT_TYPES']]
 
 
 class BaseUserForm(FlaskForm):
@@ -107,4 +111,14 @@ class TenantDomainForm(FlaskForm):
     submit = SubmitField('Add Domain')
 
 
+class TenantSelectionForm(FlaskForm):
+    types = SelectMultipleField('Tenant Types', choices=[], validators=[Optional()])
+    search = StringField('Search', validators=[Optional()])
+    submit = SubmitField('Filter')
+
+    def __init__(self, *args, **kwargs):
+        super(TenantSelectionForm, self).__init__(*args, **kwargs)
+        self.types.choices = [(t, t) for t in current_app.config['TENANT_TYPES']]
+
+
 

eveai_app/views/user_views.py

@@ -10,7 +10,7 @@ import ast
 from common.models.user import User, Tenant, Role, TenantDomain
 from common.extensions import db, security, minio_client, simple_encryption
 from common.utils.security_utils import send_confirmation_email, send_reset_email
-from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm
+from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm, TenantSelectionForm
 from common.utils.database import Database
 from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
 from common.utils.simple_encryption import generate_api_key
@@ -47,34 +47,6 @@ def tenant():
         # Handle the required attributes
         new_tenant = Tenant()
         form.populate_obj(new_tenant)
-        # new_tenant = Tenant(name=form.name.data,
-        #                     website=form.website.data,
-        #                     default_language=form.default_language.data,
-        #                     allowed_languages=form.allowed_languages.data,
-        #                     timezone=form.timezone.data,
-        #                     embedding_model=form.embedding_model.data,
-        #                     llm_model=form.llm_model.data,
-        #                     license_start_date=form.license_start_date.data,
-        #                     license_end_date=form.license_end_date.data,
-        #                     allowed_monthly_interactions=form.allowed_monthly_interactions.data,
-        #                     embed_tuning=form.embed_tuning.data,
-        #                     rag_tuning=form.rag_tuning.data)
-
-        # Handle Embedding Variables
-        new_tenant.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
-        new_tenant.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
-            if form.html_end_tags.data else []
-        new_tenant.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
-            if form.html_included_elements.data else []
-        new_tenant.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
-            if form.html_excluded_elements.data else []
-        new_tenant.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
-            if form.html_excluded_classes.data else []
-
-        current_app.logger.debug(f'html_tags: {new_tenant.html_tags},'
-                                 f'html_end_tags: {new_tenant.html_end_tags},'
-                                 f'html_included_elements: {new_tenant.html_included_elements},'
-                                 f'html_excluded_elements: {new_tenant.html_excluded_elements}')
 
         # Handle Timestamps
         timestamp = dt.now(tz.utc)
@@ -87,7 +59,7 @@ def tenant():
             db.session.commit()
         except SQLAlchemyError as e:
             current_app.logger.error(f'Failed to add tenant to database. Error: {str(e)}')
-            flash(f'Failed to add tenant to database. Error: {str(e)}')
+            flash(f'Failed to add tenant to database. Error: {str(e)}', 'danger')
             return render_template('user/tenant.html', form=form)
 
         current_app.logger.info(f"Successfully created tenant {new_tenant.id} in Database")
@@ -117,29 +89,11 @@ def edit_tenant(tenant_id):
     if request.method == 'GET':
         # Populate the form with tenant data
         form.populate_obj(tenant)
-        if tenant.html_tags:
-            form.html_tags.data = ', '.join(tenant.html_tags)
-        if tenant.html_end_tags:
-            form.html_end_tags.data = ', '.join(tenant.html_end_tags)
-        if tenant.html_included_elements:
-            form.html_included_elements.data = ', '.join(tenant.html_included_elements)
-        if tenant.html_excluded_elements:
-            form.html_excluded_elements.data = ', '.join(tenant.html_excluded_elements)
-        if tenant.html_excluded_classes:
-            form.html_excluded_classes.data = ', '.join(tenant.html_excluded_classes)
 
     if form.validate_on_submit():
+        current_app.logger.debug(f'Updating tenant {tenant_id}')
         # Populate the tenant with form data
         form.populate_obj(tenant)
-        # Then handle the special fields manually
-        tenant.html_tags = [tag.strip() for tag in form.html_tags.data.split(',') if tag.strip()]
-        tenant.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',') if tag.strip()]
-        tenant.html_included_elements = [elem.strip() for elem in form.html_included_elements.data.split(',') if
-                                         elem.strip()]
-        tenant.html_excluded_elements = [elem.strip() for elem in form.html_excluded_elements.data.split(',') if
-                                         elem.strip()]
-        tenant.html_excluded_classes = [elem.strip() for elem in form.html_excluded_classes.data.split(',') if
-                                        elem.strip()]
 
         db.session.commit()
         flash('Tenant updated successfully.', 'success')
@@ -148,9 +102,10 @@ def edit_tenant(tenant_id):
                 session['tenant'] = tenant.to_dict()
         # return redirect(url_for(f"user/tenant/tenant_id"))
     else:
+        current_app.logger.debug(f'Tenant update failed with errors: {form.errors}')
         form_validation_failed(request, form)
 
-    return render_template('user/edit_tenant.html', form=form, tenant_id=tenant_id)
+    return render_template('user/tenant.html', form=form, tenant_id=tenant_id)
 
 
 @user_bp.route('/user', methods=['GET', 'POST'])
@@ -245,20 +200,29 @@ def edit_user(user_id):
     return render_template('user/edit_user.html', form=form, user_id=user_id)
 
 
-@user_bp.route('/select_tenant')
+@user_bp.route('/select_tenant', methods=['GET', 'POST'])
 @roles_required('Super User')
 def select_tenant():
+    filter_form = TenantSelectionForm(request.form)
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
 
-    query = Tenant.query.order_by(Tenant.name)  # Fetch all tenants from the database
+    query = Tenant.query
 
-    pagination = query.paginate(page=page, per_page=per_page)
+    if filter_form.validate_on_submit():
+        if filter_form.types.data:
+            query = query.filter(Tenant.type.in_(filter_form.types.data))
+        if filter_form.search.data:
+            search = f"%{filter_form.search.data}%"
+            query = query.filter(Tenant.name.ilike(search))
+
+    query = query.order_by(Tenant.name)
+    pagination = query.paginate(page=page, per_page=per_page, error_out=False)
     tenants = pagination.items
 
-    rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', '')])
+    rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', ''), ('type', '')])
 
-    return render_template('user/select_tenant.html', rows=rows, pagination=pagination)
+    return render_template('user/select_tenant.html', rows=rows, pagination=pagination, filter_form=filter_form)
 
 
 @user_bp.route('/handle_tenant_selection', methods=['POST'])
@@ -267,10 +231,16 @@ def handle_tenant_selection():
     tenant_identification = request.form['selected_row']
     tenant_id = ast.literal_eval(tenant_identification).get('value')
     the_tenant = Tenant.query.get(tenant_id)
+
+    # set tenant information in the session
     session['tenant'] = the_tenant.to_dict()
     session['default_language'] = the_tenant.default_language
     session['embedding_model'] = the_tenant.embedding_model
     session['llm_model'] = the_tenant.llm_model
+    # remove catalog-related items from the session
+    session.pop('catalog_id', None)
+    session.pop('catalog_name', None)
+
     action = request.form['action']
 
     match action:

eveai_beat/__init__.py

@@ -0,0 +1,44 @@
+import logging
+import logging.config
+from flask import Flask
+import os
+
+from common.utils.celery_utils import make_celery, init_celery
+from config.logging_config import LOGGING
+from config.config import get_config
+
+
+def create_app(config_file=None):
+    app = Flask(__name__)
+
+    environment = os.getenv('FLASK_ENV', 'development')
+
+    match environment:
+        case 'development':
+            app.config.from_object(get_config('dev'))
+        case 'production':
+            app.config.from_object(get_config('prod'))
+        case _:
+            app.config.from_object(get_config('dev'))
+
+    logging.config.dictConfig(LOGGING)
+
+    register_extensions(app)
+
+    celery = make_celery(app.name, app.config)
+    init_celery(celery, app, is_beat=True)
+
+    from . import schedule
+    celery.conf.beat_schedule = schedule.beat_schedule
+
+    app.logger.info("EveAI Beat Scheduler Started Successfully")
+    app.logger.info("-------------------------------------------------------------------------------------------------")
+
+    return app, celery
+
+
+def register_extensions(app):
+    pass
+
+
+app, celery = create_app()

eveai_beat/schedule.py

@@ -0,0 +1,17 @@
+from celery.schedules import crontab
+
+# Define the Celery beat schedule here
+beat_schedule = {
+    'update-tenant-usages-every-hour': {
+        'task': 'update_usages',
+        'schedule': crontab(minute='0'),  # Runs every hour
+        'args': (),
+        'options': {'queue': 'entitlements'}
+    },
+    # 'send-invoices-every-month': {
+    #     'task': 'send_invoices',
+    #     'schedule': crontab(day_of_month=1, hour=0, minute=0),  # Runs on the 1st of every month
+    #     'args': ()
+    # },
+    # Add more schedules as needed
+}

eveai_chat/__init__.py

@@ -3,7 +3,7 @@ import logging.config
 from flask import Flask, jsonify
 import os
 
-from common.extensions import db, socketio, jwt, cors, session, simple_encryption
+from common.extensions import db, socketio, jwt, cors, session, simple_encryption, metrics
 from config.logging_config import LOGGING
 from eveai_chat.socket_handlers import chat_handler
 from common.utils.cors_utils import create_cors_after_request
@@ -32,17 +32,6 @@ def create_app(config_file=None):
     app.celery = make_celery(app.name, app.config)
     init_celery(app.celery, app)
 
-    # Register Blueprints
-    # register_blueprints(app)
-
-    @app.route('/ping')
-    def ping():
-        return 'pong'
-
-    @app.route('/health', methods=['GET'])
-    def health():
-        return jsonify({'status': 'ok'}), 200
-
     app.logger.info("EveAI Chat Server Started Successfully")
     app.logger.info("-------------------------------------------------------------------------------------------------")
     return app
@@ -61,8 +50,8 @@ def register_extensions(app):
                       ping_interval=app.config.get('SOCKETIO_PING_INTERVAL'),
                       )
     jwt.init_app(app)
-    # kms_client.init_app(app)
     simple_encryption.init_app(app)
+    metrics.init_app(app)
 
     # Cors setup
     cors.init_app(app, resources={r"/chat/*": {"origins": "*"}})
@@ -72,5 +61,5 @@ def register_extensions(app):
 
 
 def register_blueprints(app):
-    from .views.chat_views import chat_bp
+    from views.healthz_views import healthz_bp
-    app.register_blueprint(chat_bp)
+    app.register_blueprint(healthz_bp)

eveai_chat/socket_handlers/chat_handler.py

@@ -1,10 +1,13 @@
 import uuid
+from functools import wraps
 
 from flask_jwt_extended import create_access_token, get_jwt_identity, verify_jwt_in_request, decode_token
 from flask_socketio import emit, disconnect, join_room, leave_room
 from flask import current_app, request, session
 from sqlalchemy.exc import SQLAlchemyError
 from datetime import datetime, timedelta
+from prometheus_client import Counter, Histogram
+from time import time
 
 from common.extensions import socketio, db, simple_encryption
 from common.models.user import Tenant
@@ -12,8 +15,27 @@ from common.models.interaction import Interaction
 from common.utils.celery_utils import current_celery
 from common.utils.database import Database
 
+# Define custom metrics
+socketio_message_counter = Counter('socketio_message_count', 'Count of SocketIO messages', ['event_type'])
+socketio_message_latency = Histogram('socketio_message_latency_seconds', 'Latency of SocketIO message processing', ['event_type'])
+
+
+# Decorator to measure SocketIO events
+def track_socketio_event(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        event_type = func.__name__
+        socketio_message_counter.labels(event_type=event_type).inc()
+        start_time = time()
+        result = func(*args, **kwargs)
+        latency = time() - start_time
+        socketio_message_latency.labels(event_type=event_type).observe(latency)
+        return result
+    return wrapper
+
 
 @socketio.on('connect')
+@track_socketio_event
 def handle_connect():
     try:
         current_app.logger.debug(f'SocketIO: Connection handling started using {request.args}')
@@ -58,6 +80,7 @@ def handle_connect():
 
 
 @socketio.on('disconnect')
+@track_socketio_event
 def handle_disconnect():
     room = session.get('room')
     if room:
@@ -86,14 +109,16 @@ def handle_message(data):
         room = session.get('room')
 
         # Offload actual processing of question
-        task = current_celery.send_task('ask_question', queue='llm_interactions', args=[
+        task = current_celery.send_task('ask_question',
-            current_tenant_id,
+                                        queue='llm_interactions',
-            data['message'],
+                                        args=[
-            data['language'],
+                                            current_tenant_id,
-            session['session_id'],
+                                            data['message'],
-            data['timezone'],
+                                            data['language'],
-            room
+                                            session['session_id'],
-        ])
+                                            data['timezone'],
+                                            room
+                                        ])
         current_app.logger.debug(f'SocketIO: Message offloading for tenant {current_tenant_id}, '
                                  f'Question: {task.id}')
         response = {

eveai_chat/views/chat_views.py

@@ -1,77 +0,0 @@
-from datetime import datetime as dt, timezone as tz
-from flask import request, redirect, url_for, render_template, Blueprint, session, current_app, jsonify
-from flask_security import hash_password, roles_required, roles_accepted
-from sqlalchemy.exc import SQLAlchemyError
-from flask_jwt_extended import create_access_token, jwt_required, get_jwt_identity
-from flask_socketio import emit, join_room, leave_room
-import ast
-
-
-from common.models.user import User, Tenant
-from common.models.interaction import ChatSession, Interaction, InteractionEmbedding
-from common.models.document import Embedding
-from common.extensions import db, socketio, kms_client
-from common.utils.database import Database
-
-chat_bp = Blueprint('chat_bp', __name__, url_prefix='/chat')
-
-
-@chat_bp.route('/register_client', methods=['POST'])
-def register_client():
-    tenant_id = request.json.get('tenant_id')
-    api_key = request.json.get('api_key')
-
-    # Validate tenant_id and api_key here (e.g., check against the database)
-    if validate_tenant(tenant_id, api_key):
-        access_token = create_access_token(identity={'tenant_id': tenant_id, 'api_key': api_key})
-        current_app.logger.debug(f'Tenant Registration: Tenant {tenant_id} registered successfully')
-        return jsonify({'token': access_token}), 200
-    else:
-        current_app.logger.debug(f'Tenant Registration: Invalid tenant_id ({tenant_id}) or api_key ({api_key})')
-        return jsonify({'message': 'Invalid credentials'}), 401
-
-
-@socketio.on('connect', namespace='/chat')
-@jwt_required()
-def handle_connect():
-    current_tenant = get_jwt_identity()
-    current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} connected')
-
-
-@socketio.on('message', namespace='/chat')
-@jwt_required()
-def handle_message(data):
-    current_tenant = get_jwt_identity()
-    current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} sent a message: {data}')
-    # Store interaction in the database
-    emit('response', {'data': 'Message received'}, broadcast=True)
-
-
-def validate_tenant(tenant_id, api_key):
-    tenant = Tenant.query.get_or_404(tenant_id)
-    encrypted_api_key = ast.literal_eval(tenant.encrypted_chat_api_key)
-
-    decrypted_api_key = kms_client.decrypt_api_key(encrypted_api_key)
-
-    return decrypted_api_key == api_key
-
-
-
-# @chat_bp.route('/', methods=['GET', 'POST'])
-# def chat():
-#     return render_template('chat.html')
-#
-#
-# @chat.record_once
-# def on_register(state):
-#     # TODO: write initialisation code when the blueprint is registered (only once)
-#     # socketio.init_app(state.app)
-#     pass
-#
-#
-# @socketio.on('message', namespace='/chat')
-# def handle_message(message):
-#     # TODO: write message handling code to actually realise chat
-#     # print('Received message:', message)
-#     # socketio.emit('response', {'data': message}, namespace='/chat')
-#     pass

eveai_chat/views/healthz_views.py

@@ -0,0 +1,70 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+from eveai_chat.socket_handlers.chat_handler import socketio_message_counter, socketio_message_latency
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('ping', queue='llm_interactions')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return metrics.generate_latest()
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )
+    # Register SocketIO metrics with Prometheus
+    metrics.register(socketio_message_counter)
+    metrics.register(socketio_message_latency)

eveai_chat_workers/tasks.py

@@ -22,12 +22,23 @@ from common.models.interaction import ChatSession, Interaction, InteractionEmbed
 from common.extensions import db
 from common.utils.celery_utils import current_celery
 from common.utils.model_utils import select_model_variables, create_language_template, replace_variable_in_template
-from common.langchain.EveAIRetriever import EveAIRetriever
+from common.langchain.eveai_retriever import EveAIRetriever
-from common.langchain.EveAIHistoryRetriever import EveAIHistoryRetriever
+from common.langchain.eveai_history_retriever import EveAIHistoryRetriever
+from common.utils.business_event import BusinessEvent
+from common.utils.business_event_context import current_event
+
+
+# Healthcheck task
+@current_celery.task(name='ping', queue='llm_interactions')
+def ping():
+    return 'pong'
 
 
 def detail_question(question, language, model_variables, session_id):
-    retriever = EveAIHistoryRetriever(model_variables, session_id)
+    current_app.logger.debug(f'Detail question: {question}')
+    current_app.logger.debug(f'model_variables: {model_variables}')
+    current_app.logger.debug(f'session_id: {session_id}')
+    retriever = EveAIHistoryRetriever(model_variables=model_variables, session_id=session_id)
     llm = model_variables['llm']
     template = model_variables['history_template']
     language_template = create_language_template(template, language)
@@ -56,53 +67,50 @@ def ask_question(tenant_id, question, language, session_id, user_timezone, room)
     'interaction_id': 'interaction_id_value'
     }
     """
-    current_app.logger.info(f'ask_question: Received question for tenant {tenant_id}: {question}. Processing...')
+    with BusinessEvent("Ask Question", tenant_id=tenant_id, chat_session_id=session_id):
+        current_app.logger.info(f'ask_question: Received question for tenant {tenant_id}: {question}. Processing...')
 
-    try:
+        try:
-        # Retrieve the tenant
+            # Retrieve the tenant
-        tenant = Tenant.query.get(tenant_id)
+            tenant = Tenant.query.get(tenant_id)
-        if not tenant:
+            if not tenant:
-            raise Exception(f'Tenant {tenant_id} not found.')
+                raise Exception(f'Tenant {tenant_id} not found.')
 
-        # Ensure we are working in the correct database schema
+            # Ensure we are working in the correct database schema
-        Database(tenant_id).switch_schema()
+            Database(tenant_id).switch_schema()
 
-        # Ensure we have a session to story history
+            # Ensure we have a session to story history
-        chat_session = ChatSession.query.filter_by(session_id=session_id).first()
+            chat_session = ChatSession.query.filter_by(session_id=session_id).first()
-        if not chat_session:
+            if not chat_session:
-            try:
+                try:
-                chat_session = ChatSession()
+                    chat_session = ChatSession()
-                chat_session.session_id = session_id
+                    chat_session.session_id = session_id
-                chat_session.session_start = dt.now(tz.utc)
+                    chat_session.session_start = dt.now(tz.utc)
-                chat_session.timezone = user_timezone
+                    chat_session.timezone = user_timezone
-                db.session.add(chat_session)
+                    db.session.add(chat_session)
-                db.session.commit()
+                    db.session.commit()
-            except SQLAlchemyError as e:
+                except SQLAlchemyError as e:
-                current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}')
+                    current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}')
-                raise
+                    raise
 
-        if tenant.rag_tuning:
+            with current_event.create_span("RAG Answer"):
-            current_app.rag_tuning_logger.debug(f'Received question for tenant {tenant_id}:\n{question}. Processing...')
+                result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session)
-            current_app.rag_tuning_logger.debug(f'Tenant Information: \n{tenant.to_dict()}')
+                result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
-            current_app.rag_tuning_logger.debug(f'===================================================================')
-            current_app.rag_tuning_logger.debug(f'===================================================================')
-
-        result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session)
-        result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
-        result['interaction_id'] = interaction.id
-        result['room'] = room  # Include the room in the result
-
-        if result['insufficient_info']:
-            if 'LLM' in tenant.fallback_algorithms:
-                result, interaction = answer_using_llm(question, language, tenant, chat_session)
-                result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name']
                 result['interaction_id'] = interaction.id
                 result['room'] = room  # Include the room in the result
 
-        return result
+            if result['insufficient_info']:
-    except Exception as e:
+                if 'LLM' in tenant.fallback_algorithms:
-        current_app.logger.error(f'ask_question: Error processing question: {e}')
+                    with current_event.create_span("Fallback Algorithm LLM"):
-        raise
+                        result, interaction = answer_using_llm(question, language, tenant, chat_session)
+                        result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name']
+                        result['interaction_id'] = interaction.id
+                        result['room'] = room  # Include the room in the result
+
+            return result
+        except Exception as e:
+            current_app.logger.error(f'ask_question: Error processing question: {e}')
+            raise
 
 
 def answer_using_tenant_rag(question, language, tenant, chat_session):
@@ -122,92 +130,93 @@ def answer_using_tenant_rag(question, language, tenant, chat_session):
     # Langchain debugging if required
     # set_debug(True)
 
-    detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
+    with current_event.create_span("Detail Question"):
-    current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
+        detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
-    if tenant.rag_tuning:
+        if model_variables['rag_tuning']:
-        current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.')
+            current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.')
-        current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
-    new_interaction.detailed_question = detailed_question
-    new_interaction.detailed_question_at = dt.now(tz.utc)
-
-    retriever = EveAIRetriever(model_variables, tenant_info)
-    llm = model_variables['llm']
-    template = model_variables['rag_template']
-    language_template = create_language_template(template, language)
-    full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
-    rag_prompt = ChatPromptTemplate.from_template(full_template)
-    setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()})
-    if tenant.rag_tuning:
-        current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.')
-        current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
-
-    new_interaction_embeddings = []
-    if not model_variables['cited_answer_cls']:  # The model doesn't support structured feedback
-        output_parser = StrOutputParser()
-
-        chain = setup_and_retrieval | rag_prompt | llm | output_parser
-
-        # Invoke the chain with the actual question
-        answer = chain.invoke(detailed_question)
-        new_interaction.answer = answer
-        result = {
-            'answer': answer,
-            'citations': [],
-            'insufficient_info': False
-        }
-
-    else:  # The model supports structured feedback
-        structured_llm = llm.with_structured_output(model_variables['cited_answer_cls'])
-
-        chain = setup_and_retrieval | rag_prompt | structured_llm
-
-        result = chain.invoke(detailed_question).dict()
-        current_app.logger.debug(f'ask_question: result answer: {result['answer']}')
-        current_app.logger.debug(f'ask_question: result citations: {result["citations"]}')
-        current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
-        if tenant.rag_tuning:
-            current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}')
-            current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}')
-            current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
             current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
-        new_interaction.answer = result['answer']
+        new_interaction.detailed_question = detailed_question
+        new_interaction.detailed_question_at = dt.now(tz.utc)
 
-        # Filter out the existing Embedding IDs
+    with current_event.create_span("Generate Answer using RAG"):
-        given_embedding_ids = [int(emb_id) for emb_id in result['citations']]
+        retriever = EveAIRetriever(model_variables, tenant_info)
-        embeddings = (
+        llm = model_variables['llm']
-            db.session.query(Embedding)
+        template = model_variables['rag_template']
-            .filter(Embedding.id.in_(given_embedding_ids))
+        language_template = create_language_template(template, language)
-            .all()
+        full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
-        )
+        rag_prompt = ChatPromptTemplate.from_template(full_template)
-        existing_embedding_ids = [emb.id for emb in embeddings]
+        setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()})
-        urls = list(set(emb.document_version.url for emb in embeddings))
+        if model_variables['rag_tuning']:
-        if tenant.rag_tuning:
+            current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.')
-            current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n')
-            current_app.rag_tuning_logger.debug(f'{urls}')
             current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
 
-        for emb_id in existing_embedding_ids:
+        new_interaction_embeddings = []
-            new_interaction_embedding = InteractionEmbedding(embedding_id=emb_id)
+        if not model_variables['cited_answer_cls']:  # The model doesn't support structured feedback
-            new_interaction_embedding.interaction = new_interaction
+            output_parser = StrOutputParser()
-            new_interaction_embeddings.append(new_interaction_embedding)
 
-        result['citations'] = urls
+            chain = setup_and_retrieval | rag_prompt | llm | output_parser
 
-    # Disable langchain debugging if set above.
+            # Invoke the chain with the actual question
-    # set_debug(False)
+            answer = chain.invoke(detailed_question)
+            new_interaction.answer = answer
+            result = {
+                'answer': answer,
+                'citations': [],
+                'insufficient_info': False
+            }
 
-    new_interaction.answer_at = dt.now(tz.utc)
+        else:  # The model supports structured feedback
-    chat_session.session_end = dt.now(tz.utc)
+            structured_llm = llm.with_structured_output(model_variables['cited_answer_cls'])
 
-    try:
+            chain = setup_and_retrieval | rag_prompt | structured_llm
-        db.session.add(chat_session)
+
-        db.session.add(new_interaction)
+            result = chain.invoke(detailed_question).dict()
-        db.session.add_all(new_interaction_embeddings)
+            current_app.logger.debug(f'ask_question: result answer: {result['answer']}')
-        db.session.commit()
+            current_app.logger.debug(f'ask_question: result citations: {result["citations"]}')
-        return result, new_interaction
+            current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
-    except SQLAlchemyError as e:
+            if model_variables['rag_tuning']:
-        current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
+                current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}')
-        raise
+                current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}')
+                current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
+                current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
+            new_interaction.answer = result['answer']
+
+            # Filter out the existing Embedding IDs
+            given_embedding_ids = [int(emb_id) for emb_id in result['citations']]
+            embeddings = (
+                db.session.query(Embedding)
+                .filter(Embedding.id.in_(given_embedding_ids))
+                .all()
+            )
+            existing_embedding_ids = [emb.id for emb in embeddings]
+            urls = list(set(emb.document_version.url for emb in embeddings))
+            if model_variables['rag_tuning']:
+                current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n')
+                current_app.rag_tuning_logger.debug(f'{urls}')
+                current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
+
+            for emb_id in existing_embedding_ids:
+                new_interaction_embedding = InteractionEmbedding(embedding_id=emb_id)
+                new_interaction_embedding.interaction = new_interaction
+                new_interaction_embeddings.append(new_interaction_embedding)
+
+            result['citations'] = urls
+
+        # Disable langchain debugging if set above.
+        # set_debug(False)
+
+        new_interaction.answer_at = dt.now(tz.utc)
+        chat_session.session_end = dt.now(tz.utc)
+
+        try:
+            db.session.add(chat_session)
+            db.session.add(new_interaction)
+            db.session.add_all(new_interaction_embeddings)
+            db.session.commit()
+            return result, new_interaction
+        except SQLAlchemyError as e:
+            current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
+            raise
 
 
 def answer_using_llm(question, language, tenant, chat_session):
@@ -227,47 +236,49 @@ def answer_using_llm(question, language, tenant, chat_session):
     # Langchain debugging if required
     # set_debug(True)
 
-    detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
+    with current_event.create_span("Detail Question"):
-    current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
+        detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
-    new_interaction.detailed_question = detailed_question
+        current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
-    new_interaction.detailed_question_at = dt.now(tz.utc)
+        new_interaction.detailed_question = detailed_question
+        new_interaction.detailed_question_at = dt.now(tz.utc)
 
-    retriever = EveAIRetriever(model_variables, tenant_info)
+    with current_event.create_span("Detail Answer using LLM"):
-    llm = model_variables['llm_no_rag']
+        retriever = EveAIRetriever(model_variables, tenant_info)
-    template = model_variables['encyclopedia_template']
+        llm = model_variables['llm_no_rag']
-    language_template = create_language_template(template, language)
+        template = model_variables['encyclopedia_template']
-    rag_prompt = ChatPromptTemplate.from_template(language_template)
+        language_template = create_language_template(template, language)
-    setup = RunnablePassthrough()
+        rag_prompt = ChatPromptTemplate.from_template(language_template)
-    output_parser = StrOutputParser()
+        setup = RunnablePassthrough()
+        output_parser = StrOutputParser()
 
-    new_interaction_embeddings = []
+        new_interaction_embeddings = []
 
-    chain = setup | rag_prompt | llm | output_parser
+        chain = setup | rag_prompt | llm | output_parser
-    input_question = {"question": detailed_question}
+        input_question = {"question": detailed_question}
 
-    # Invoke the chain with the actual question
+        # Invoke the chain with the actual question
-    answer = chain.invoke(input_question)
+        answer = chain.invoke(input_question)
-    new_interaction.answer = answer
+        new_interaction.answer = answer
-    result = {
+        result = {
-        'answer': answer,
+            'answer': answer,
-        'citations': [],
+            'citations': [],
-        'insufficient_info': False
+            'insufficient_info': False
-    }
+        }
 
-    # Disable langchain debugging if set above.
+        # Disable langchain debugging if set above.
-    # set_debug(False)
+        # set_debug(False)
 
-    new_interaction.answer_at = dt.now(tz.utc)
+        new_interaction.answer_at = dt.now(tz.utc)
-    chat_session.session_end = dt.now(tz.utc)
+        chat_session.session_end = dt.now(tz.utc)
 
-    try:
+        try:
-        db.session.add(chat_session)
+            db.session.add(chat_session)
-        db.session.add(new_interaction)
+            db.session.add(new_interaction)
-        db.session.commit()
+            db.session.commit()
-        return result, new_interaction
+            return result, new_interaction
-    except SQLAlchemyError as e:
+        except SQLAlchemyError as e:
-        current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
+            current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
-        raise
+            raise
 
 
 def tasks_ping():

eveai_entitlements/__init__.py

@@ -0,0 +1,44 @@
+import logging
+import logging.config
+from flask import Flask
+import os
+
+from common.utils.celery_utils import make_celery, init_celery
+from common.extensions import db, minio_client
+from config.logging_config import LOGGING
+from config.config import get_config
+
+
+def create_app(config_file=None):
+    app = Flask(__name__)
+
+    environment = os.getenv('FLASK_ENV', 'development')
+
+    match environment:
+        case 'development':
+            app.config.from_object(get_config('dev'))
+        case 'production':
+            app.config.from_object(get_config('prod'))
+        case _:
+            app.config.from_object(get_config('dev'))
+
+    logging.config.dictConfig(LOGGING)
+
+    register_extensions(app)
+
+    celery = make_celery(app.name, app.config)
+    init_celery(celery, app)
+
+    from . import tasks
+
+    app.logger.info("EveAI Entitlements Server Started Successfully")
+    app.logger.info("-------------------------------------------------------------------------------------------------")
+
+    return app, celery
+
+
+def register_extensions(app):
+    db.init_app(app)
+
+
+app, celery = create_app()

eveai_entitlements/tasks.py

@@ -0,0 +1,253 @@
+import io
+import os
+from datetime import datetime as dt, timezone as tz, datetime
+
+from celery import states
+from dateutil.relativedelta import relativedelta
+from flask import current_app
+from sqlalchemy import or_, and_, text
+from sqlalchemy.exc import SQLAlchemyError
+from common.extensions import db
+from common.models.user import Tenant
+from common.models.entitlements import BusinessEventLog, LicenseUsage, License
+from common.utils.celery_utils import current_celery
+from common.utils.eveai_exceptions import EveAINoLicenseForTenant, EveAIException
+from common.utils.database import Database
+
+
+# Healthcheck task
+@current_celery.task(name='ping', queue='entitlements')
+def ping():
+    return 'pong'
+
+
+@current_celery.task(name='update_usages', queue='entitlements')
+def update_usages():
+    current_timestamp = dt.now(tz.utc)
+    tenant_ids = get_all_tenant_ids()
+
+    # List to collect all errors
+    error_list = []
+
+    for tenant_id in tenant_ids:
+        try:
+            Database(tenant_id).switch_schema()
+            check_and_create_license_usage_for_tenant(tenant_id)
+            tenant = Tenant.query.get(tenant_id)
+            if tenant.storage_dirty:
+                recalculate_storage_for_tenant(tenant)
+            logs = get_logs_for_processing(tenant_id, current_timestamp)
+            if not logs:
+                continue    # If no logs to be processed, continu to the next tenant
+
+            # Get the min and max timestamp from the logs
+            min_timestamp = min(log.timestamp for log in logs)
+            max_timestamp = max(log.timestamp for log in logs)
+
+            # Retrieve relevant LicenseUsage records
+            current_app.logger.debug(f"Searching relevant usages for tenant {tenant_id}")
+            license_usages = get_relevant_license_usages(db.session, tenant_id, min_timestamp, max_timestamp)
+            current_app.logger.debug(f"Found {license_usages}, end searching relevant usages for tenant {tenant_id}")
+
+            # Split logs based on LicenseUsage periods
+            current_app.logger.debug(f"Splitting usages for tenant {tenant_id}")
+            logs_by_usage = split_logs_by_license_usage(logs, license_usages)
+            current_app.logger.debug(f"Found {logs_by_usage}, end splitting logs for tenant {tenant_id}")
+
+            # Now you can process logs for each LicenseUsage
+            for license_usage_id, logs in logs_by_usage.items():
+                current_app.logger.debug(f"Processing logs for usage id {license_usage_id} for tenant {tenant_id}")
+                process_logs_for_license_usage(tenant_id, license_usage_id, logs)
+                current_app.logger.debug(f"Finished processing logs for tenant {tenant_id}")
+        except Exception as e:
+            error = f"Usage Calculation error for Tenant {tenant_id}: {e}"
+            error_list.append(error)
+            current_app.logger.error(error)
+            continue
+
+    if error_list:
+        raise Exception('\n'.join(error_list))
+
+    return "Update Usages taks completed successfully"
+
+
+def get_all_tenant_ids():
+    tenant_ids = db.session.query(Tenant.id).all()
+    return [tenant_id[0] for tenant_id in tenant_ids]  # Extract tenant_id from tuples
+
+
+def check_and_create_license_usage_for_tenant(tenant_id):
+    current_date = dt.now(tz.utc).date()
+    license_usages = (db.session.query(LicenseUsage)
+                      .filter_by(tenant_id=tenant_id)
+                      .filter(and_(LicenseUsage.period_start_date <= current_date,
+                                   LicenseUsage.period_end_date >= current_date))
+                      .all())
+    if not license_usages:
+        active_license = (db.session.query(License).filter_by(tenant_id=tenant_id)
+                          .filter(and_(License.start_date <= current_date,
+                                       License.end_date >= current_date))
+                          .one_or_none())
+        if not active_license:
+            current_app.logger.error(f"No License defined for {tenant_id}. "
+                                     f"Impossible to calculate license usage.")
+            raise EveAINoLicenseForTenant(message=f"No License defined for {tenant_id}. "
+                                                  f"Impossible to calculate license usage.")
+
+        start_date, end_date = calculate_valid_period(current_date, active_license.start_date)
+        new_license_usage = LicenseUsage(period_start_date=start_date,
+                                         period_end_date=end_date,
+                                         license_id=active_license.id,
+                                         tenant_id=tenant_id
+                                         )
+        try:
+            db.session.add(new_license_usage)
+            db.session.commit()
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            current_app.logger.error(f"Error trying to create new license usage for tenant {tenant_id}. "
+                                     f"Error: {str(e)}")
+            raise e
+
+
+def calculate_valid_period(given_date, original_start_date):
+    # Ensure both dates are of datetime.date type
+    if isinstance(given_date, datetime):
+        given_date = given_date.date()
+    if isinstance(original_start_date, datetime):
+        original_start_date = original_start_date.date()
+
+    # Step 1: Find the most recent start_date less than or equal to given_date
+    start_date = original_start_date
+    while start_date <= given_date:
+        next_start_date = start_date + relativedelta(months=1)
+        if next_start_date > given_date:
+            break
+        start_date = next_start_date
+
+    # Step 2: Calculate the end_date for this period
+    end_date = start_date + relativedelta(months=1, days=-1)
+
+    # Ensure the given date falls within the period
+    if start_date <= given_date <= end_date:
+        return start_date, end_date
+    else:
+        raise ValueError("Given date does not fall within a valid period.")
+
+
+def get_logs_for_processing(tenant_id, end_time_stamp):
+    return (db.session.query(BusinessEventLog).filter(
+        BusinessEventLog.tenant_id == tenant_id,
+        BusinessEventLog.license_usage_id == None,
+        BusinessEventLog.timestamp <= end_time_stamp,
+    ).all())
+
+
+def get_relevant_license_usages(session, tenant_id, min_timestamp, max_timestamp):
+    # Fetch LicenseUsage records where the log timestamps fall between period_start_date and period_end_date
+    return session.query(LicenseUsage).filter(
+        LicenseUsage.tenant_id == tenant_id,
+        LicenseUsage.period_start_date <= max_timestamp.date(),
+        LicenseUsage.period_end_date >= min_timestamp.date()
+    ).order_by(LicenseUsage.period_start_date).all()
+
+
+def split_logs_by_license_usage(logs, license_usages):
+    # Dictionary to hold logs categorized by LicenseUsage
+    logs_by_usage = {lu.id: [] for lu in license_usages}
+
+    for log in logs:
+        # Find the corresponding LicenseUsage for each log based on the timestamp
+        for license_usage in license_usages:
+            if license_usage.period_start_date <= log.timestamp.date() <= license_usage.period_end_date:
+                logs_by_usage[license_usage.id].append(log)
+                break
+
+    return logs_by_usage
+
+
+def process_logs_for_license_usage(tenant_id, license_usage_id, logs):
+    # Retrieve the LicenseUsage record
+    license_usage = db.session.query(LicenseUsage).filter_by(id=license_usage_id).first()
+
+    if not license_usage:
+        raise ValueError(f"LicenseUsage with id {license_usage_id} not found.")
+
+    # Initialize variables to accumulate usage data
+    embedding_mb_used = 0
+    embedding_prompt_tokens_used = 0
+    embedding_completion_tokens_used = 0
+    embedding_total_tokens_used = 0
+    interaction_prompt_tokens_used = 0
+    interaction_completion_tokens_used = 0
+    interaction_total_tokens_used = 0
+
+    # Process each log
+    for log in logs:
+        # Case for 'Create Embeddings' event
+        if log.event_type == 'Create Embeddings':
+            if log.message == 'Starting Trace for Create Embeddings':
+                embedding_mb_used += log.document_version_file_size
+            elif log.message == 'Final LLM Metrics':
+                embedding_prompt_tokens_used += log.llm_metrics_prompt_tokens
+                embedding_completion_tokens_used += log.llm_metrics_completion_tokens
+                embedding_total_tokens_used += log.llm_metrics_total_tokens
+
+        # Case for 'Ask Question' event
+        elif log.event_type == 'Ask Question':
+            if log.message == 'Final LLM Metrics':
+                interaction_prompt_tokens_used += log.llm_metrics_prompt_tokens
+                interaction_completion_tokens_used += log.llm_metrics_completion_tokens
+                interaction_total_tokens_used += log.llm_metrics_total_tokens
+
+        # Mark the log as processed by setting the license_usage_id
+        log.license_usage_id = license_usage_id
+
+    # Update the LicenseUsage record with the accumulated values
+    license_usage.embedding_mb_used += embedding_mb_used
+    license_usage.embedding_prompt_tokens_used += embedding_prompt_tokens_used
+    license_usage.embedding_completion_tokens_used += embedding_completion_tokens_used
+    license_usage.embedding_total_tokens_used += embedding_total_tokens_used
+    license_usage.interaction_prompt_tokens_used += interaction_prompt_tokens_used
+    license_usage.interaction_completion_tokens_used += interaction_completion_tokens_used
+    license_usage.interaction_total_tokens_used += interaction_total_tokens_used
+
+    current_app.logger.debug(f"Processed logs for license usage {license_usage.id}:\n{license_usage}")
+
+    # Commit the updates to the LicenseUsage and log records
+    try:
+        db.session.add(license_usage)
+        for log in logs:
+            db.session.add(log)
+        db.session.commit()
+    except SQLAlchemyError as e:
+        db.session.rollback()
+        current_app.logger.error(f"Error trying to update license usage and logs for tenant {tenant_id}: {e}")
+        raise e
+
+
+def recalculate_storage_for_tenant(tenant):
+    # Perform a SUM operation to get the total file size from document_versions
+    total_storage = db.session.execute(text(f"""
+        SELECT SUM(file_size) 
+        FROM document_version
+    """)).scalar()
+    current_app.logger.debug(f"Recalculating storage for tenant {tenant} - Total storage: {total_storage}")
+
+    # Update the LicenseUsage with the recalculated storage
+    license_usage = db.session.query(LicenseUsage).filter_by(tenant_id=tenant.id).first()
+    license_usage.storage_mb_used = total_storage
+
+    # Reset the dirty flag after recalculating
+    tenant.storage_dirty = False
+
+    # Commit the changes
+    try:
+        db.session.add(tenant)
+        db.session.add(license_usage)
+        db.session.commit()
+    except SQLAlchemyError as e:
+        db.session.rollback()
+        current_app.logger.error(f"Error trying to update tenant {tenant.id} for Dirty Storage. ")
+
+

eveai_workers/Processors/audio_processor.py

@@ -1,101 +1,151 @@
 import io
 import os
+import time
+
+import psutil
 from pydub import AudioSegment
 import tempfile
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
-from common.utils.model_utils import create_language_template
-from .processor import Processor
 import subprocess
 
+from .transcription_processor import TranscriptionProcessor
+from common.utils.business_event_context import current_event
 
-class AudioProcessor(Processor):
+
+class AudioProcessor(TranscriptionProcessor):
     def __init__(self, tenant, model_variables, document_version):
         super().__init__(tenant, model_variables, document_version)
         self.transcription_client = model_variables['transcription_client']
         self.transcription_model = model_variables['transcription_model']
         self.ffmpeg_path = 'ffmpeg'
+        self.max_compression_duration = model_variables['max_compression_duration']
+        self.max_transcription_duration = model_variables['max_transcription_duration']
+        self.compression_cpu_limit = model_variables.get('compression_cpu_limit', 50)  # CPU usage limit in percentage
+        self.compression_process_delay = model_variables.get('compression_process_delay', 0.1)  # Delay between processing chunks in seconds
+        self.file_type = document_version.file_type
 
+    def _get_transcription(self):
+        file_data = minio_client.download_document_file(
+            self.tenant.id,
+            self.document_version.bucket_name,
+            self.document_version.object_name,
+        )
 
-    def process(self):
+        with current_event.create_span("Audio Compression"):
-        self._log("Starting Audio processing")
-        try:
-            file_data = minio_client.download_document_file(
-                self.tenant.id,
-                self.document_version.doc_id,
-                self.document_version.language,
-                self.document_version.id,
-                self.document_version.file_name
-            )
-
             compressed_audio = self._compress_audio(file_data)
+        with current_event.create_span("Audio Transcription"):
             transcription = self._transcribe_audio(compressed_audio)
-            markdown, title = self._generate_markdown_from_transcription(transcription)
 
-            self._save_markdown(markdown)
+        return transcription
-            self._log("Finished processing Audio")
-            return markdown, title
-        except Exception as e:
-            self._log(f"Error processing Audio: {str(e)}", level='error')
-            raise
 
     def _compress_audio(self, audio_data):
         self._log("Compressing audio")
-        with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{self.document_version.file_type}') as temp_input:
-            temp_input.write(audio_data)
-            temp_input.flush()
 
-            # Use a unique filename for the output to avoid conflicts
+        with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{self.document_version.file_type}') as temp_file:
-            output_filename = f'compressed_{os.urandom(8).hex()}.mp3'
+            temp_file.write(audio_data)
-            output_path = os.path.join(tempfile.gettempdir(), output_filename)
+            temp_file_path = temp_file.name
 
-            try:
+        try:
-                result = subprocess.run(
+            self._log("Creating AudioSegment from file")
-                    [self.ffmpeg_path, '-y', '-i', temp_input.name, '-b:a', '64k', '-f', 'mp3', output_path],
+            audio_info = AudioSegment.from_file(temp_file_path, format=self.document_version.file_type)
-                    capture_output=True,
+            self._log("Finished creating AudioSegment from file")
-                    text=True,
+            total_duration = len(audio_info)
-                    check=True
+            self._log(f"Audio duration: {total_duration / 1000} seconds")
+
+            segment_length = self.max_compression_duration * 1000  # Convert to milliseconds
+            total_chunks = (total_duration + segment_length - 1) // segment_length
+
+            compressed_segments = AudioSegment.empty()
+
+            for i in range(total_chunks):
+                self._log(f"Compressing segment {i + 1} of {total_chunks}")
+
+                start_time = i * segment_length
+                end_time = min((i + 1) * segment_length, total_duration)
+
+                chunk = AudioSegment.from_file(
+                    temp_file_path,
+                    format=self.document_version.file_type,
+                    start_second=start_time / 1000,
+                    duration=(end_time - start_time) / 1000
                 )
 
-                with open(output_path, 'rb') as f:
+                compressed_chunk = self._compress_segment(chunk)
-                    compressed_data = f.read()
+                compressed_segments += compressed_chunk
 
-                # Save compressed audio to MinIO
+                time.sleep(self.compression_process_delay)
-                compressed_filename = f"{self.document_version.id}_compressed.mp3"
+
+            # Save compressed audio to MinIO
+            compressed_filename = f"{self.document_version.id}_compressed.mp3"
+            with io.BytesIO() as compressed_buffer:
+                compressed_segments.export(compressed_buffer, format="mp3")
+                compressed_buffer.seek(0)
                 minio_client.upload_document_file(
                     self.tenant.id,
                     self.document_version.doc_id,
                     self.document_version.language,
                     self.document_version.id,
                     compressed_filename,
-                    compressed_data
+                    compressed_buffer.read()
                 )
-                self._log(f"Saved compressed audio to MinIO: {compressed_filename}")
+            self._log(f"Saved compressed audio to MinIO: {compressed_filename}")
 
-                return compressed_data
+            return compressed_segments
 
-            except subprocess.CalledProcessError as e:
+        except Exception as e:
-                error_message = f"Compression failed: {e.stderr}"
+            self._log(f"Error during audio processing: {str(e)}", level='error')
-                self._log(error_message, level='error')
+            raise
-                raise Exception(error_message)
+        finally:
+            os.unlink(temp_file_path)  # Ensure the temporary file is deleted
 
-            finally:
+    def _compress_segment(self, audio_segment):
-                # Clean up temporary files
+        with io.BytesIO() as segment_buffer:
-                os.unlink(temp_input.name)
+            audio_segment.export(segment_buffer, format="wav")
-                if os.path.exists(output_path):
+            segment_buffer.seek(0)
-                    os.unlink(output_path)
+
+            with io.BytesIO() as output_buffer:
+                command = [
+                    'nice', '-n', '19',
+                    'ffmpeg',
+                    '-i', 'pipe:0',
+                    '-ar', '16000',
+                    '-ac', '1',
+                    '-b:a', '32k',
+                    '-filter:a', 'loudnorm',
+                    '-f', 'mp3',
+                    'pipe:1'
+                ]
+
+                process = psutil.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+                stdout, stderr = process.communicate(input=segment_buffer.read())
+
+                if process.returncode != 0:
+                    self._log(f"FFmpeg error: {stderr.decode()}", level='error')
+                    raise Exception("FFmpeg compression failed")
+
+                output_buffer.write(stdout)
+                output_buffer.seek(0)
+                compressed_segment = AudioSegment.from_mp3(output_buffer)
+
+        return compressed_segment
 
     def _transcribe_audio(self, audio_data):
         self._log("Starting audio transcription")
-        audio = AudioSegment.from_file(io.BytesIO(audio_data), format="mp3")
+        # audio = AudioSegment.from_file(io.BytesIO(audio_data), format="mp3")
+        audio = audio_data
 
-        segment_length = 10 * 60 * 1000  # 10 minutes in milliseconds
+        segment_length = self.max_transcription_duration * 1000  # calculate milliseconds
         transcriptions = []
+        total_chunks = len(audio) // segment_length + 1
 
         for i, chunk in enumerate(audio[::segment_length]):
-            self._log(f'Processing chunk {i + 1} of {len(audio) // segment_length + 1}')
+            self._log(f'Processing chunk {i + 1} of {total_chunks}')
+            segment_duration = 0
+            if i == total_chunks - 1:
+                segment_duration = (len(audio) % segment_length) // 1000
+            else:
+                segment_duration = self.max_transcription_duration
 
             with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as temp_audio:
                 chunk.export(temp_audio.name, format="mp3")
@@ -111,11 +161,12 @@ class AudioProcessor(Processor):
                         audio_file.seek(0)  # Reset file pointer to the beginning
 
                         self._log("Calling transcription API")
-                        transcription = self.transcription_client.audio.transcriptions.create(
+                        transcription = self.model_variables.transcribe(
                             file=audio_file,
                             model=self.transcription_model,
                             language=self.document_version.language,
                             response_format='verbose_json',
+                            duration=segment_duration,
                         )
                         self._log("Transcription API call completed")
 
@@ -159,29 +210,3 @@ class AudioProcessor(Processor):
 
         return full_transcription
 
-    def _generate_markdown_from_transcription(self, transcription):
-        self._log("Generating markdown from transcription")
-        llm = self.model_variables['llm']
-        template = self.model_variables['transcript_template']
-        language_template = create_language_template(template, self.document_version.language)
-        transcript_prompt = ChatPromptTemplate.from_template(language_template)
-        setup = RunnablePassthrough()
-        output_parser = StrOutputParser()
-
-        chain = setup | transcript_prompt | llm | output_parser
-
-        input_transcript = {'transcript': transcription}
-        markdown = chain.invoke(input_transcript)
-
-        # Extract title from the markdown
-        title = self._extract_title_from_markdown(markdown)
-
-        return markdown, title
-
-    def _extract_title_from_markdown(self, markdown):
-        # Simple extraction of the first header as the title
-        lines = markdown.split('\n')
-        for line in lines:
-            if line.startswith('# '):
-                return line[2:].strip()
-        return "Untitled Audio Transcription"

eveai_workers/Processors/html_processor.py

@@ -5,6 +5,7 @@ from langchain_core.runnables import RunnablePassthrough
 from common.extensions import db, minio_client
 from common.utils.model_utils import create_language_template
 from .processor import Processor
+from common.utils.business_event_context import current_event
 
 
 class HTMLProcessor(Processor):
@@ -14,21 +15,25 @@ class HTMLProcessor(Processor):
         self.html_end_tags = model_variables['html_end_tags']
         self.html_included_elements = model_variables['html_included_elements']
         self.html_excluded_elements = model_variables['html_excluded_elements']
+        self.html_excluded_classes = model_variables['html_excluded_classes']
+        self.chunk_size = model_variables['processing_chunk_size']  # Adjust this based on your LLM's optimal input size
+        self.chunk_overlap = model_variables[
+            'processing_chunk_overlap']  # Adjust for context preservation between chunks
 
     def process(self):
         self._log("Starting HTML processing")
         try:
             file_data = minio_client.download_document_file(
                 self.tenant.id,
-                self.document_version.doc_id,
+                self.document_version.bucket_name,
-                self.document_version.language,
+                self.document_version.object_name,
-                self.document_version.id,
-                self.document_version.file_name
             )
             html_content = file_data.decode('utf-8')
 
-            extracted_html, title = self._parse_html(html_content)
+            with current_event.create_span("HTML Content Extraction"):
-            markdown = self._generate_markdown_from_html(extracted_html)
+                extracted_html, title = self._parse_html(html_content)
+            with current_event.create_span("Markdown Generation"):
+                markdown = self._generate_markdown_from_html(extracted_html)
 
             self._save_markdown(markdown)
             self._log("Finished processing HTML")
@@ -41,7 +46,7 @@ class HTMLProcessor(Processor):
         self._log(f'Parsing HTML for tenant {self.tenant.id}')
         soup = BeautifulSoup(html_content, 'html.parser')
         extracted_html = ''
-        excluded_classes = self._parse_excluded_classes(self.tenant.html_excluded_classes)
+        excluded_classes = self._parse_excluded_classes(self.html_excluded_classes)
 
         if self.html_included_elements:
             elements_to_parse = soup.find_all(self.html_included_elements)
@@ -70,7 +75,7 @@ class HTMLProcessor(Processor):
         chain = setup | parse_prompt | llm | output_parser
 
         soup = BeautifulSoup(html_content, 'lxml')
-        chunks = self._split_content(soup)
+        chunks = self._split_content(soup, self.chunk_size)
 
         markdown_chunks = []
         for chunk in chunks:

eveai_workers/Processors/pdf_processor.py

@@ -10,35 +10,35 @@ from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
 from common.utils.model_utils import create_language_template
 from .processor import Processor
+from common.utils.business_event_context import current_event
 
 
 class PDFProcessor(Processor):
     def __init__(self, tenant, model_variables, document_version):
         super().__init__(tenant, model_variables, document_version)
         # PDF-specific initialization
-        self.chunk_size = model_variables['PDF_chunk_size']
+        self.chunk_size = model_variables['processing_chunk_size']
-        self.chunk_overlap = model_variables['PDF_chunk_overlap']
+        self.chunk_overlap = model_variables['processing_chunk_overlap']
-        self.min_chunk_size = model_variables['PDF_min_chunk_size']
+        self.min_chunk_size = model_variables['processing_min_chunk_size']
-        self.max_chunk_size = model_variables['PDF_max_chunk_size']
+        self.max_chunk_size = model_variables['processing_max_chunk_size']
 
     def process(self):
         self._log("Starting PDF processing")
         try:
             file_data = minio_client.download_document_file(
                 self.tenant.id,
-                self.document_version.doc_id,
+                self.document_version.bucket_name,
-                self.document_version.language,
+                self.document_version.object_name,
-                self.document_version.id,
-                self.document_version.file_name
             )
 
-            extracted_content = self._extract_content(file_data)
+            with current_event.create_span("PDF Extraction"):
-            structured_content, title = self._structure_content(extracted_content)
+                extracted_content = self._extract_content(file_data)
+                structured_content, title = self._structure_content(extracted_content)
 
-            llm_chunks = self._split_content_for_llm(structured_content)
+            with current_event.create_span("Markdown Generation"):
-            markdown = self._process_chunks_with_llm(llm_chunks)
+                llm_chunks = self._split_content_for_llm(structured_content)
-
+                markdown = self._process_chunks_with_llm(llm_chunks)
-            self._save_markdown(markdown)
+                self._save_markdown(markdown)
             self._log("Finished processing PDF")
             return markdown, title
         except Exception as e:
@@ -228,12 +228,7 @@ class PDFProcessor(Processor):
         for chunk in chunks:
             input = {"pdf_content": chunk}
             result = chain.invoke(input)
-            # Remove Markdown code block delimiters if present
+            result = self._clean_markdown(result)
-            result = result.strip()
-            if result.startswith("```markdown"):
-                result = result[len("```markdown"):].strip()
-            if result.endswith("```"):
-                result = result[:-3].strip()
             markdown_chunks.append(result)
 
         return "\n\n".join(markdown_chunks)