- Portkey log retrieval started

- flower container added (dev and prod)

.gitignore

@@ -41,3 +41,4 @@ migrations/.DS_Store
 migrations/public/.DS_Store
 scripts/.DS_Store
 scripts/__pycache__/run_eveai_app.cpython-312.pyc
+/eveai_repo.txt

.repopackignore

@@ -0,0 +1,21 @@
+# Add patterns to ignore here, one per line
+# Example:
+# *.log
+# tmp/
+logs/
+nginx/static/assets/fonts/
+nginx/static/assets/img/
+nginx/static/assets/js/
+nginx/static/scss/
+patched_packages/
+migrations/
+*material*
+*nucleo*
+*package*
+nginx/mime.types
+*.gitignore*
+.python-version
+.repopackignore
+repopack.config.json
+
+

CHANGELOG.md

@@ -24,7 +24,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 - In case of vulnerabilities.
-- 
+
+## [1.0.8-alfa] - 2024-09-12
+
+### Added
+- Tenant type defined to allow for active, inactive, demo ... tenants
+- Search and filtering functionality on Tenants
+- Implementation of health checks (1st version)
+- Provision for Prometheus monitoring (no implementation yet)
+- Refine audio_processor and srt_processor to reduce duplicate code and support larger files
+- Introduction of repopack to reason in LLMs about the code
+
+### Fixed
+- Refine audio_processor and srt_processor to reduce duplicate code and support larger files
+
+## [1.0.7-alfa] - 2024-09-12
+
+### Added
+- Full Document API allowing for creation, updating and invalidation of documents.
+- Metadata fields (JSON) added to DocumentVersion, allowing end-users to add structured information
+- Wordpress plugin eveai_sync to synchronize Wordpress content with EveAI
+
+### Fixed
+- Maximal deduplication of code between views and api in document_utils.py
+
 ## [1.0.6-alfa] - 2024-09-03
 
 ### Fixed

common/extensions.py

@@ -10,8 +10,8 @@ from flask_jwt_extended import JWTManager
 from flask_session import Session
 from flask_wtf import CSRFProtect
 from flask_restx import Api
+from prometheus_flask_exporter import PrometheusMetrics
 
-from .utils.nginx_utils import prefixed_url_for
 from .utils.simple_encryption import SimpleEncryption
 from .utils.minio_utils import MinioClient
 
@@ -31,3 +31,4 @@ session = Session()
 api_rest = Api()
 simple_encryption = SimpleEncryption()
 minio_client = MinioClient()
+metrics = PrometheusMetrics.for_app_factory()

common/langchain/eveai_history_retriever.py

@@ -1,23 +1,31 @@
 from langchain_core.retrievers import BaseRetriever
 from sqlalchemy import asc
 from sqlalchemy.exc import SQLAlchemyError
-from pydantic import BaseModel, Field
+from pydantic import Field, BaseModel, PrivateAttr
 from typing import Any, Dict
 from flask import current_app
 
 from common.extensions import db
 from common.models.interaction import ChatSession, Interaction
-from common.utils.datetime_utils import get_date_in_timezone
+from common.utils.model_utils import ModelVariables
 
 
-class EveAIHistoryRetriever(BaseRetriever):
-    model_variables: Dict[str, Any] = Field(...)
-    session_id: str = Field(...)
+class EveAIHistoryRetriever(BaseRetriever, BaseModel):
+    _model_variables: ModelVariables = PrivateAttr()
+    _session_id: str = PrivateAttr()
 
-    def __init__(self, model_variables: Dict[str, Any], session_id: str):
+    def __init__(self, model_variables: ModelVariables, session_id: str):
         super().__init__()
-        self.model_variables = model_variables
-        self.session_id = session_id
+        self._model_variables = model_variables
+        self._session_id = session_id
+
+    @property
+    def model_variables(self) -> ModelVariables:
+        return self._model_variables
+
+    @property
+    def session_id(self) -> str:
+        return self._session_id
 
     def _get_relevant_documents(self, query: str):
         current_app.logger.debug(f'Retrieving history of interactions for query: {query}')

common/langchain/eveai_retriever.py

@@ -1,30 +1,39 @@
 from langchain_core.retrievers import BaseRetriever
 from sqlalchemy import func, and_, or_, desc
 from sqlalchemy.exc import SQLAlchemyError
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, PrivateAttr
 from typing import Any, Dict
 from flask import current_app
 
 from common.extensions import db
 from common.models.document import Document, DocumentVersion
 from common.utils.datetime_utils import get_date_in_timezone
+from common.utils.model_utils import ModelVariables
 
 
-class EveAIRetriever(BaseRetriever):
-    model_variables: Dict[str, Any] = Field(...)
-    tenant_info: Dict[str, Any] = Field(...)
+class EveAIRetriever(BaseRetriever, BaseModel):
+    _model_variables: ModelVariables = PrivateAttr()
+    _tenant_info: Dict[str, Any] = PrivateAttr()
 
-    def __init__(self, model_variables: Dict[str, Any], tenant_info: Dict[str, Any]):
+    def __init__(self, model_variables: ModelVariables, tenant_info: Dict[str, Any]):
         super().__init__()
-        self.model_variables = model_variables
-        self.tenant_info = tenant_info
+        current_app.logger.debug(f'Model variables type: {type(model_variables)}')
+        self._model_variables = model_variables
+        self._tenant_info = tenant_info
+
+    @property
+    def model_variables(self) -> ModelVariables:
+        return self._model_variables
+
+    @property
+    def tenant_info(self) -> Dict[str, Any]:
+        return self._tenant_info
 
     def _get_relevant_documents(self, query: str):
-
-
-
         current_app.logger.debug(f'Retrieving relevant documents for query: {query}')
         query_embedding = self._get_query_embedding(query)
+        current_app.logger.debug(f'Model Variables Private: {type(self._model_variables)}')
+        current_app.logger.debug(f'Model Variables Property: {type(self.model_variables)}')
         db_class = self.model_variables['embedding_db_model']
         similarity_threshold = self.model_variables['similarity_threshold']
         k = self.model_variables['k']

common/models/monitoring.py

@@ -0,0 +1,21 @@
+from common.extensions import db
+
+
+class BusinessEventLog(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = {'schema': 'public'}
+
+    id = db.Column(db.Integer, primary_key=True)
+    timestamp = db.Column(db.DateTime, nullable=False)
+    event_type = db.Column(db.String(50), nullable=False)
+    tenant_id = db.Column(db.Integer, nullable=False)
+    trace_id = db.Column(db.String(50), nullable=False)
+    span_id = db.Column(db.String(50))
+    span_name = db.Column(db.String(50))
+    parent_span_id = db.Column(db.String(50))
+    document_version_id = db.Column(db.Integer)
+    chat_session_id = db.Column(db.String(50))
+    interaction_id = db.Column(db.Integer)
+    environment = db.Column(db.String(20))
+    message = db.Column(db.Text)
+    # Add any other fields relevant for invoicing or warnings

common/models/user.py

@@ -2,7 +2,6 @@ from common.extensions import db
 from flask_security import UserMixin, RoleMixin
 from sqlalchemy.dialects.postgresql import ARRAY
 import sqlalchemy as sa
-from sqlalchemy import CheckConstraint
 
 
 class Tenant(db.Model):
@@ -21,6 +20,7 @@ class Tenant(db.Model):
     website = db.Column(db.String(255), nullable=True)
     timezone = db.Column(db.String(50), nullable=True, default='UTC')
     rag_context = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(20), nullable=True, server_default='Active')
 
     # language information
     default_language = db.Column(db.String(2), nullable=True)
@@ -56,7 +56,6 @@ class Tenant(db.Model):
     encrypted_chat_api_key = db.Column(db.String(500), nullable=True)
     encrypted_api_key = db.Column(db.String(500), nullable=True)
 
-
     # Tuning enablers
     embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
     rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
@@ -75,6 +74,7 @@ class Tenant(db.Model):
             'website': self.website,
             'timezone': self.timezone,
             'rag_context': self.rag_context,
+            'type': self.type,
             'default_language': self.default_language,
             'allowed_languages': self.allowed_languages,
             'embedding_model': self.embedding_model,

common/utils/business_event.py

@@ -0,0 +1,114 @@
+import os
+import uuid
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Dict, Any, Optional
+from datetime import datetime as dt, timezone as tz
+from portkey_ai import Portkey, Config
+import logging
+
+from .business_event_context import BusinessEventContext
+from common.models.monitoring import BusinessEventLog
+from common.extensions import db
+
+
+class BusinessEvent:
+    # The BusinessEvent class itself is a context manager, but it doesn't use the @contextmanager decorator.
+    # Instead, it defines __enter__ and __exit__ methods explicitly. This is because we're doing something a bit more
+    # complex - we're interacting with the BusinessEventContext and the _business_event_stack.
+
+    def __init__(self, event_type: str, tenant_id: int, **kwargs):
+        self.event_type = event_type
+        self.tenant_id = tenant_id
+        self.trace_id = str(uuid.uuid4())
+        self.span_id = None
+        self.span_name = None
+        self.parent_span_id = None
+        self.document_version_id = kwargs.get('document_version_id')
+        self.chat_session_id = kwargs.get('chat_session_id')
+        self.interaction_id = kwargs.get('interaction_id')
+        self.environment = os.environ.get("FLASK_ENV", "development")
+        self.span_counter = 0
+        self.spans = []
+
+    def update_attribute(self, attribute: str, value: any):
+        if hasattr(self, attribute):
+            setattr(self, attribute, value)
+        else:
+            raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'")
+
+    @contextmanager
+    def create_span(self, span_name: str):
+        # The create_span method is designed to be used as a context manager. We want to perform some actions when
+        # entering the span (like setting the span ID and name) and some actions when exiting the span (like removing
+        # these temporary attributes). The @contextmanager decorator allows us to write this method in a way that
+        # clearly separates the "entry" and "exit" logic, with the yield statement in between.
+
+        parent_span_id = self.span_id
+        self.span_counter += 1
+        new_span_id = str(uuid.uuid4())
+
+        # Save the current span info
+        self.spans.append((self.span_id, self.span_name, self.parent_span_id))
+
+        # Set the new span info
+        self.span_id = new_span_id
+        self.span_name = span_name
+        self.parent_span_id = parent_span_id
+
+        self.log(f"Starting span {span_name}")
+
+        try:
+            yield
+        finally:
+            self.log(f"Ending span {span_name}")
+            # Restore the previous span info
+            if self.spans:
+                self.span_id, self.span_name, self.parent_span_id = self.spans.pop()
+            else:
+                self.span_id = None
+                self.span_name = None
+                self.parent_span_id = None
+
+    def log(self, message: str, level: str = 'info'):
+        logger = logging.getLogger('business_events')
+        log_data = {
+            'event_type': self.event_type,
+            'tenant_id': self.tenant_id,
+            'trace_id': self.trace_id,
+            'span_id': self.span_id,
+            'span_name': self.span_name,
+            'parent_span_id': self.parent_span_id,
+            'document_version_id': self.document_version_id,
+            'chat_session_id': self.chat_session_id,
+            'interaction_id': self.interaction_id,
+            'environment': self.environment
+        }
+        # log to Graylog
+        getattr(logger, level)(message, extra=log_data)
+
+        # Log to database
+        event_log = BusinessEventLog(
+            timestamp=dt.now(tz=tz.utc),
+            event_type=self.event_type,
+            tenant_id=self.tenant_id,
+            trace_id=self.trace_id,
+            span_id=self.span_id,
+            span_name=self.span_name,
+            parent_span_id=self.parent_span_id,
+            document_version_id=self.document_version_id,
+            chat_session_id=self.chat_session_id,
+            interaction_id=self.interaction_id,
+            environment=self.environment,
+            message=message
+        )
+        db.session.add(event_log)
+        db.session.commit()
+
+    def __enter__(self):
+        self.log(f'Starting Trace for {self.event_type}')
+        return BusinessEventContext(self).__enter__()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.log(f'Ending Trace for {self.event_type}')
+        return BusinessEventContext(self).__exit__(exc_type, exc_val, exc_tb)

common/utils/business_event_context.py

@@ -0,0 +1,25 @@
+from werkzeug.local import LocalProxy, LocalStack
+
+_business_event_stack = LocalStack()
+
+
+def _get_current_event():
+    top = _business_event_stack.top
+    if top is None:
+        raise RuntimeError("No business event context found. Are you sure you're in a business event?")
+    return top
+
+
+current_event = LocalProxy(_get_current_event)
+
+
+class BusinessEventContext:
+    def __init__(self, event):
+        self.event = event
+
+    def __enter__(self):
+        _business_event_stack.push(self.event)
+        return self.event
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        _business_event_stack.pop()

common/utils/cors_utils.py

@@ -23,6 +23,14 @@ def cors_after_request(response, prefix):
     current_app.logger.debug(f'request.args: {request.args}')
     current_app.logger.debug(f'request is json?: {request.is_json}')
 
+    # Exclude health checks from checks
+    if request.path.startswith('/healthz') or request.path.startswith('/_healthz'):
+        current_app.logger.debug('Skipping CORS headers for health checks')
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', '*')
+        response.headers.add('Access-Control-Allow-Methods', '*')
+        return response
+
     tenant_id = None
     allowed_origins = []
 

common/utils/model_utils.py

@@ -5,14 +5,16 @@ from flask import current_app
 from langchain_openai import OpenAIEmbeddings, ChatOpenAI
 from langchain_anthropic import ChatAnthropic
 from langchain_core.pydantic_v1 import BaseModel, Field
-from langchain.prompts import ChatPromptTemplate
-import ast
-from typing import List
+from typing import List, Any, Iterator
+from collections.abc import MutableMapping
 from openai import OpenAI
-# from groq import Groq
 from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL
+from portkey_ai.langchain.portkey_langchain_callback_handler import LangchainCallbackHandler
 
 from common.models.document import EmbeddingSmallOpenAI, EmbeddingLargeOpenAI
+from common.models.user import Tenant
+from config.model_config import MODEL_CONFIG
+from common.utils.business_event_context import current_event
 
 
 class CitedAnswer(BaseModel):
@@ -36,180 +38,264 @@ def set_language_prompt_template(cls, language_prompt):
     cls.__doc__ = language_prompt
 
 
+class ModelVariables(MutableMapping):
+    def __init__(self, tenant: Tenant):
+        self.tenant = tenant
+        self._variables = self._initialize_variables()
+        self._embedding_model = None
+        self._llm = None
+        self._llm_no_rag = None
+        self._transcription_client = None
+        self._prompt_templates = {}
+        self._embedding_db_model = None
+
+    def _initialize_variables(self):
+        variables = {}
+
+        # We initialize the variables that are available knowing the tenant. For the other, we will apply 'lazy loading'
+        variables['k'] = self.tenant.es_k or 5
+        variables['similarity_threshold'] = self.tenant.es_similarity_threshold or 0.7
+        variables['RAG_temperature'] = self.tenant.chat_RAG_temperature or 0.3
+        variables['no_RAG_temperature'] = self.tenant.chat_no_RAG_temperature or 0.5
+        variables['embed_tuning'] = self.tenant.embed_tuning or False
+        variables['rag_tuning'] = self.tenant.rag_tuning or False
+        variables['rag_context'] = self.tenant.rag_context or " "
+
+        # Set HTML Chunking Variables
+        variables['html_tags'] = self.tenant.html_tags
+        variables['html_end_tags'] = self.tenant.html_end_tags
+        variables['html_included_elements'] = self.tenant.html_included_elements
+        variables['html_excluded_elements'] = self.tenant.html_excluded_elements
+        variables['html_excluded_classes'] = self.tenant.html_excluded_classes
+
+        # Set Chunk Size variables
+        variables['min_chunk_size'] = self.tenant.min_chunk_size
+        variables['max_chunk_size'] = self.tenant.max_chunk_size
+
+        # Set model providers
+        variables['embedding_provider'], variables['embedding_model'] = self.tenant.embedding_model.rsplit('.', 1)
+        variables['llm_provider'], variables['llm_model'] = self.tenant.llm_model.rsplit('.', 1)
+        variables["templates"] = current_app.config['PROMPT_TEMPLATES'][(f"{variables['llm_provider']}."
+                                                                         f"{variables['llm_model']}")]
+        current_app.logger.info(f"Loaded prompt templates: \n")
+        current_app.logger.info(f"{variables['templates']}")
+
+        # Set model-specific configurations
+        model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {})
+        variables.update(model_config)
+
+        variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][self.tenant.llm_model]
+
+        if variables['tool_calling_supported']:
+            variables['cited_answer_cls'] = CitedAnswer
+
+        return variables
+
+    @property
+    def embedding_model(self):
+        portkey_metadata = self.get_portkey_metadata()
+
+        portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
+                                        provider=self._variables['embedding_provider'],
+                                        metadata=portkey_metadata,
+                                        trace_id=current_event.trace_id,
+                                        span_id=current_event.span_id,
+                                        span_name=current_event.span_name,
+                                        parent_span_id=current_event.parent_span_id
+                                        )
+        api_key = os.getenv('OPENAI_API_KEY')
+        model = self._variables['embedding_model']
+        self._embedding_model = OpenAIEmbeddings(api_key=api_key,
+                                                 model=model,
+                                                 base_url=PORTKEY_GATEWAY_URL,
+                                                 default_headers=portkey_headers)
+        self._embedding_db_model = EmbeddingSmallOpenAI \
+            if model == 'text-embedding-3-small' \
+            else EmbeddingLargeOpenAI
+
+        return self._embedding_model
+
+    @property
+    def llm(self):
+        portkey_headers = self.get_portkey_headers_for_llm()
+        api_key = self.get_api_key_for_llm()
+        self._llm = ChatOpenAI(api_key=api_key,
+                               model=self._variables['llm_model'],
+                               temperature=self._variables['RAG_temperature'],
+                               base_url=PORTKEY_GATEWAY_URL,
+                               default_headers=portkey_headers)
+        return self._llm
+
+    @property
+    def llm_no_rag(self):
+        portkey_headers = self.get_portkey_headers_for_llm()
+        api_key = self.get_api_key_for_llm()
+        self._llm_no_rag = ChatOpenAI(api_key=api_key,
+                                      model=self._variables['llm_model'],
+                                      temperature=self._variables['RAG_temperature'],
+                                      base_url=PORTKEY_GATEWAY_URL,
+                                      default_headers=portkey_headers)
+        return self._llm_no_rag
+
+    def get_portkey_headers_for_llm(self):
+        portkey_metadata = self.get_portkey_metadata()
+        portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
+                                        metadata=portkey_metadata,
+                                        provider=self._variables['llm_provider'],
+                                        trace_id=current_event.trace_id,
+                                        span_id=current_event.span_id,
+                                        span_name=current_event.span_name,
+                                        parent_span_id=current_event.parent_span_id
+                                        )
+        return portkey_headers
+
+    def get_portkey_metadata(self):
+        environment = os.getenv('FLASK_ENV', 'development')
+        portkey_metadata = {'tenant_id': str(self.tenant.id),
+                            'environment': environment,
+                            'trace_id': current_event.trace_id,
+                            'span_id': current_event.span_id,
+                            'span_name': current_event.span_name,
+                            'parent_span_id': current_event.parent_span_id,
+                            }
+        return portkey_metadata
+
+    def get_api_key_for_llm(self):
+        if self._variables['llm_provider'] == 'openai':
+            api_key = os.getenv('OPENAI_API_KEY')
+        else: # self._variables['llm_provider'] == 'anthropic'
+            api_key = os.getenv('ANTHROPIC_API_KEY')
+
+        return api_key
+
+    # def _initialize_llm(self):
+    #
+    #
+    #     if self._variables['llm_provider'] == 'openai':
+    #         portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
+    #                                         metadata=portkey_metadata,
+    #                                         provider='openai')
+    #
+    #         self._llm = ChatOpenAI(api_key=api_key,
+    #                                model=self._variables['llm_model'],
+    #                                temperature=self._variables['RAG_temperature'],
+    #                                base_url=PORTKEY_GATEWAY_URL,
+    #                                default_headers=portkey_headers)
+    #         self._llm_no_rag = ChatOpenAI(api_key=api_key,
+    #                                       model=self._variables['llm_model'],
+    #                                       temperature=self._variables['no_RAG_temperature'],
+    #                                       base_url=PORTKEY_GATEWAY_URL,
+    #                                       default_headers=portkey_headers)
+    #         self._variables['tool_calling_supported'] = self._variables['llm_model'] in ['gpt-4o', 'gpt-4o-mini']
+    #     elif self._variables['llm_provider'] == 'anthropic':
+    #         api_key = os.getenv('ANTHROPIC_API_KEY')
+    #         llm_model_ext = os.getenv('ANTHROPIC_LLM_VERSIONS', {}).get(self._variables['llm_model'])
+    #         self._llm = ChatAnthropic(api_key=api_key,
+    #                                   model=llm_model_ext,
+    #                                   temperature=self._variables['RAG_temperature'])
+    #         self._llm_no_rag = ChatAnthropic(api_key=api_key,
+    #                                          model=llm_model_ext,
+    #                                          temperature=self._variables['RAG_temperature'])
+    #         self._variables['tool_calling_supported'] = True
+    #     else:
+    #         raise ValueError(f"Invalid chat provider: {self._variables['llm_provider']}")
+
+    @property
+    def transcription_client(self):
+        environment = os.getenv('FLASK_ENV', 'development')
+        portkey_metadata = self.get_portkey_metadata()
+        portkey_headers = createHeaders(api_key=os.getenv('PORTKEY_API_KEY'),
+                                        metadata=portkey_metadata,
+                                        provider='openai',
+                                        trace_id=current_event.trace_id,
+                                        span_id=current_event.span_id,
+                                        span_name=current_event.span_name,
+                                        parent_span_id=current_event.parent_span_id
+                                        )
+        api_key = os.getenv('OPENAI_API_KEY')
+        self._transcription_client = OpenAI(api_key=api_key,
+                                            base_url=PORTKEY_GATEWAY_URL,
+                                            default_headers=portkey_headers)
+        self._variables['transcription_model'] = 'whisper-1'
+        return self._transcription_client
+
+    @property
+    def embedding_db_model(self):
+        if self._embedding_db_model is None:
+            self._embedding_db_model = self.get_embedding_db_model()
+        return self._embedding_db_model
+
+    def get_embedding_db_model(self):
+        current_app.logger.debug("In get_embedding_db_model")
+        if self._embedding_db_model is None:
+            self._embedding_db_model = EmbeddingSmallOpenAI \
+                if self._variables['embedding_model'] == 'text-embedding-3-small' \
+                else EmbeddingLargeOpenAI
+        current_app.logger.debug(f"Embedding DB Model: {self._embedding_db_model}")
+        return self._embedding_db_model
+
+    def get_prompt_template(self, template_name: str) -> str:
+        current_app.logger.info(f"Getting prompt template for {template_name}")
+        if template_name not in self._prompt_templates:
+            self._prompt_templates[template_name] = self._load_prompt_template(template_name)
+        return self._prompt_templates[template_name]
+
+    def _load_prompt_template(self, template_name: str) -> str:
+        # In the future, this method will make an API call to Portkey
+        # For now, we'll simulate it with a placeholder implementation
+        # You can replace this with your current prompt loading logic
+        return self._variables['templates'][template_name]
+
+    def __getitem__(self, key: str) -> Any:
+        current_app.logger.debug(f"ModelVariables: Getting {key}")
+        # Support older template names (suffix = _template)
+        if key.endswith('_template'):
+            key = key[:-len('_template')]
+            current_app.logger.debug(f"ModelVariables: Getting modified {key}")
+        if key == 'embedding_model':
+            return self.embedding_model
+        elif key == 'embedding_db_model':
+            return self.embedding_db_model
+        elif key == 'llm':
+            return self.llm
+        elif key == 'llm_no_rag':
+            return self.llm_no_rag
+        elif key == 'transcription_client':
+            return self.transcription_client
+        elif key in self._variables.get('prompt_templates', []):
+            return self.get_prompt_template(key)
+        return self._variables.get(key)
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._variables[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self._variables[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._variables)
+
+    def __len__(self):
+        return len(self._variables)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        return self.__getitem__(key) or default
+
+    def update(self, **kwargs) -> None:
+        self._variables.update(kwargs)
+
+    def items(self):
+        return self._variables.items()
+
+    def keys(self):
+        return self._variables.keys()
+
+    def values(self):
+        return self._variables.values()
+
+
 def select_model_variables(tenant):
-    embedding_provider = tenant.embedding_model.rsplit('.', 1)[0]
-    embedding_model = tenant.embedding_model.rsplit('.', 1)[1]
-
-    llm_provider = tenant.llm_model.rsplit('.', 1)[0]
-    llm_model = tenant.llm_model.rsplit('.', 1)[1]
-
-    # Set model variables
-    model_variables = {}
-    if tenant.es_k:
-        model_variables['k'] = tenant.es_k
-    else:
-        model_variables['k'] = 5
-
-    if tenant.es_similarity_threshold:
-        model_variables['similarity_threshold'] = tenant.es_similarity_threshold
-    else:
-        model_variables['similarity_threshold'] = 0.7
-
-    if tenant.chat_RAG_temperature:
-        model_variables['RAG_temperature'] = tenant.chat_RAG_temperature
-    else:
-        model_variables['RAG_temperature'] = 0.3
-
-    if tenant.chat_no_RAG_temperature:
-        model_variables['no_RAG_temperature'] = tenant.chat_no_RAG_temperature
-    else:
-        model_variables['no_RAG_temperature'] = 0.5
-
-    # Set Tuning variables
-    if tenant.embed_tuning:
-        model_variables['embed_tuning'] = tenant.embed_tuning
-    else:
-        model_variables['embed_tuning'] = False
-
-    if tenant.rag_tuning:
-        model_variables['rag_tuning'] = tenant.rag_tuning
-    else:
-        model_variables['rag_tuning'] = False
-
-    if tenant.rag_context:
-        model_variables['rag_context'] = tenant.rag_context
-    else:
-        model_variables['rag_context'] = " "
-
-    # Set HTML Chunking Variables
-    model_variables['html_tags'] = tenant.html_tags
-    model_variables['html_end_tags'] = tenant.html_end_tags
-    model_variables['html_included_elements'] = tenant.html_included_elements
-    model_variables['html_excluded_elements'] = tenant.html_excluded_elements
-    model_variables['html_excluded_classes'] = tenant.html_excluded_classes
-
-    # Set Chunk Size variables
-    model_variables['min_chunk_size'] = tenant.min_chunk_size
-    model_variables['max_chunk_size'] = tenant.max_chunk_size
-
-    environment = os.getenv('FLASK_ENV', 'development')
-    portkey_metadata = {'tenant_id': str(tenant.id), 'environment': environment}
-
-    # Set Embedding variables
-    match embedding_provider:
-        case 'openai':
-            portkey_headers = createHeaders(api_key=current_app.config.get('PORTKEY_API_KEY'),
-                                            provider='openai',
-                                            metadata=portkey_metadata)
-            match embedding_model:
-                case 'text-embedding-3-small':
-                    api_key = current_app.config.get('OPENAI_API_KEY')
-                    model_variables['embedding_model'] = OpenAIEmbeddings(api_key=api_key,
-                                                                          model='text-embedding-3-small',
-                                                                          base_url=PORTKEY_GATEWAY_URL,
-                                                                          default_headers=portkey_headers
-                                                                          )
-                    model_variables['embedding_db_model'] = EmbeddingSmallOpenAI
-                case 'text-embedding-3-large':
-                    api_key = current_app.config.get('OPENAI_API_KEY')
-                    model_variables['embedding_model'] = OpenAIEmbeddings(api_key=api_key,
-                                                                          model='text-embedding-3-large',
-                                                                          base_url=PORTKEY_GATEWAY_URL,
-                                                                          default_headers=portkey_headers
-                                                                          )
-                    model_variables['embedding_db_model'] = EmbeddingLargeOpenAI
-                case _:
-                    raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                                    f'error: Invalid embedding model')
-        case _:
-            raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                            f'error: Invalid embedding provider')
-
-    # Set Chat model variables
-    match llm_provider:
-        case 'openai':
-            portkey_headers = createHeaders(api_key=current_app.config.get('PORTKEY_API_KEY'),
-                                            metadata=portkey_metadata,
-                                            provider='openai')
-            tool_calling_supported = False
-            api_key = current_app.config.get('OPENAI_API_KEY')
-            model_variables['llm'] = ChatOpenAI(api_key=api_key,
-                                                model=llm_model,
-                                                temperature=model_variables['RAG_temperature'],
-                                                base_url=PORTKEY_GATEWAY_URL,
-                                                default_headers=portkey_headers)
-            model_variables['llm_no_rag'] = ChatOpenAI(api_key=api_key,
-                                                       model=llm_model,
-                                                       temperature=model_variables['no_RAG_temperature'],
-                                                       base_url=PORTKEY_GATEWAY_URL,
-                                                       default_headers=portkey_headers)
-            tool_calling_supported = False
-            match llm_model:
-                case 'gpt-4o' | 'gpt-4o-mini':
-                    tool_calling_supported = True
-                    PDF_chunk_size = 10000
-                    PDF_chunk_overlap = 200
-                    PDF_min_chunk_size = 8000
-                    PDF_max_chunk_size = 12000
-                case _:
-                    raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                                    f'error: Invalid chat model')
-        case 'anthropic':
-            api_key = current_app.config.get('ANTHROPIC_API_KEY')
-            # Anthropic does not have the same 'generic' model names as OpenAI
-            llm_model_ext = current_app.config.get('ANTHROPIC_LLM_VERSIONS').get(llm_model)
-            model_variables['llm'] = ChatAnthropic(api_key=api_key,
-                                                   model=llm_model_ext,
-                                                   temperature=model_variables['RAG_temperature'])
-            model_variables['llm_no_rag'] = ChatAnthropic(api_key=api_key,
-                                                          model=llm_model_ext,
-                                                          temperature=model_variables['RAG_temperature'])
-            tool_calling_supported = True
-            PDF_chunk_size = 10000
-            PDF_chunk_overlap = 200
-            PDF_min_chunk_size = 8000
-            PDF_max_chunk_size = 12000
-        case _:
-            raise Exception(f'Error setting model variables for tenant {tenant.id} '
-                            f'error: Invalid chat provider')
-
-    model_variables['PDF_chunk_size'] = PDF_chunk_size
-    model_variables['PDF_chunk_overlap'] = PDF_chunk_overlap
-    model_variables['PDF_min_chunk_size'] = PDF_min_chunk_size
-    model_variables['PDF_max_chunk_size'] = PDF_max_chunk_size
-
-    if tool_calling_supported:
-        model_variables['cited_answer_cls'] = CitedAnswer
-
-    templates = current_app.config['PROMPT_TEMPLATES'][f'{llm_provider}.{llm_model}']
-    model_variables['summary_template'] = templates['summary']
-    model_variables['rag_template'] = templates['rag']
-    model_variables['history_template'] = templates['history']
-    model_variables['encyclopedia_template'] = templates['encyclopedia']
-    model_variables['transcript_template'] = templates['transcript']
-    model_variables['html_parse_template'] = templates['html_parse']
-    model_variables['pdf_parse_template'] = templates['pdf_parse']
-
-    model_variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][tenant.llm_model]
-
-    # Transcription Client Variables.
-    # Using Groq
-    # api_key = current_app.config.get('GROQ_API_KEY')
-    # model_variables['transcription_client'] = Groq(api_key=api_key)
-    # model_variables['transcription_model'] = 'whisper-large-v3'
-
-    # Using OpenAI for transcriptions
-    portkey_metadata = {'tenant_id': str(tenant.id)}
-    portkey_headers = createHeaders(api_key=current_app.config.get('PORTKEY_API_KEY'),
-                                    metadata=portkey_metadata,
-                                    provider='openai'
-                                    )
-    api_key = current_app.config.get('OPENAI_API_KEY')
-    model_variables['transcription_client'] = OpenAI(api_key=api_key,
-                                                     base_url=PORTKEY_GATEWAY_URL,
-                                                     default_headers=portkey_headers)
-    model_variables['transcription_model'] = 'whisper-1'
-
+    model_variables = ModelVariables(tenant=tenant)
     return model_variables
 
 

common/utils/portkey_utils.py

@@ -0,0 +1,99 @@
+import requests
+import json
+from typing import Optional
+
+# Define a function to make the GET request
+def get_metadata_grouped_data(
+    api_key: str,
+    metadata_key: str,
+    time_of_generation_min: Optional[str] = None,
+    time_of_generation_max: Optional[str] = None,
+    total_units_min: Optional[int] = None,
+    total_units_max: Optional[int] = None,
+    cost_min: Optional[float] = None,
+    cost_max: Optional[float] = None,
+    prompt_token_min: Optional[int] = None,
+    prompt_token_max: Optional[int] = None,
+    completion_token_min: Optional[int] = None,
+    completion_token_max: Optional[int] = None,
+    status_code: Optional[str] = None,
+    weighted_feedback_min: Optional[float] = None,
+    weighted_feedback_max: Optional[float] = None,
+    virtual_keys: Optional[str] = None,
+    configs: Optional[str] = None,
+    workspace_slug: Optional[str] = None,
+    api_key_ids: Optional[str] = None,
+    current_page: Optional[int] = 1,
+    page_size: Optional[int] = 20,
+    metadata: Optional[str] = None,
+    ai_org_model: Optional[str] = None,
+    trace_id: Optional[str] = None,
+    span_id: Optional[str] = None,
+):
+    url = f"https://api.portkey.ai/v1/analytics/groups/metadata/{metadata_key}"
+
+    # Set up query parameters
+    params = {
+        "time_of_generation_min": time_of_generation_min,
+        "time_of_generation_max": time_of_generation_max,
+        "total_units_min": total_units_min,
+        "total_units_max": total_units_max,
+        "cost_min": cost_min,
+        "cost_max": cost_max,
+        "prompt_token_min": prompt_token_min,
+        "prompt_token_max": prompt_token_max,
+        "completion_token_min": completion_token_min,
+        "completion_token_max": completion_token_max,
+        "status_code": status_code,
+        "weighted_feedback_min": weighted_feedback_min,
+        "weighted_feedback_max": weighted_feedback_max,
+        "virtual_keys": virtual_keys,
+        "configs": configs,
+        "workspace_slug": workspace_slug,
+        "api_key_ids": api_key_ids,
+        "current_page": current_page,
+        "page_size": page_size,
+        "metadata": metadata,
+        "ai_org_model": ai_org_model,
+        "trace_id": trace_id,
+        "span_id": span_id,
+    }
+
+    # Remove any keys with None values
+    params = {k: v for k, v in params.items() if v is not None}
+
+    # Set up the headers
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+
+    # Make the GET request
+    response = requests.get(url, headers=headers, params=params)
+
+    # Check for successful response
+    if response.status_code == 200:
+        return response.json()  # Return JSON data
+    else:
+        response.raise_for_status()  # Raise an exception for errors
+
+# Example usage
+# Replace 'your_api_key' and 'your_metadata_key' with actual values
+api_key = 'your_api_key'
+metadata_key = 'your_metadata_key'
+
+try:
+    data = get_metadata_grouped_data(
+        api_key=api_key,
+        metadata_key=metadata_key,
+        time_of_generation_min="2024-08-23T15:50:23+05:30",
+        time_of_generation_max="2024-09-23T15:50:23+05:30",
+        total_units_min=100,
+        total_units_max=1000,
+        cost_min=10,
+        cost_max=100,
+        status_code="200,201"
+    )
+    print(json.dumps(data, indent=4))
+except Exception as e:
+    print(f"Error occurred: {str(e)}")

common/utils/view_assistants.py

@@ -1,4 +1,4 @@
-from flask import flash
+from flask import flash, current_app
 
 
 def prepare_table(model_objects, column_names):
@@ -44,7 +44,8 @@ def form_validation_failed(request, form):
         for fieldName, errorMessages in form.errors.items():
             for err in errorMessages:
                 flash(f"Error in {fieldName}: {err}", 'danger')
+                current_app.logger.debug(f"Error in {fieldName}: {err}", 'danger')
 
 
 def form_to_dict(form):
-    return {field.name: field.data for field in form if field.name != 'csrf_token' and hasattr(field, 'data')}
+    return {field.name: field.data for field in form if field.name != 'csrf_token' and hasattr(field, 'data')}

config/config.py

@@ -137,9 +137,15 @@ class Config(object):
     MAIL_PASSWORD = environ.get('MAIL_PASSWORD')
     MAIL_DEFAULT_SENDER = ('eveAI Admin', MAIL_USERNAME)
 
+    # Langsmith settings
+    LANGCHAIN_TRACING_V2 = True
+    LANGCHAIN_ENDPOINT = 'https://api.smith.langchain.com'
+    LANGCHAIN_PROJECT = "eveai"
+
+
     SUPPORTED_FILE_TYPES = ['pdf', 'html', 'md', 'txt', 'mp3', 'mp4', 'ogg', 'srt']
 
-
+    TENANT_TYPES = ['Active', 'Demo', 'Inactive', 'Test']
 
 
 class DevConfig(Config):

config/gc_sa_eveai.json

@@ -1,13 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "eveai-420711",
-  "private_key_id": "e666408e75793321a6134243628346722a71b3a6",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaGTXCWpq08YD1\nOW4z+gncOlB7T/EIiEwsZgMp6pyUrNioGfiI9YN+uVR0nsUSmFf1YyerRgX7RqD5\nRc7T/OuX8iIvmloK3g7CaFezcVrjnBKcg/QsjDAt/OO3DTk4vykDlh/Kqxx73Jdv\nFH9YSV2H7ToWqIE8CTDnqe8vQS7Bq995c9fPlues31MgndRFg3CFkH0ldfZ4aGm3\n1RnBDyC+9SPQW9e7CJgNN9PWTmOT51Zyy5IRuV5OWePMQaGLVmCo5zNc/EHZEVRu\n1hxJPHL3NNmkYDY8tye8uHgjsAkv8QuwIuUSqnqjoo1/Yg+P0+9GCpePOAJRNxJS\n0YpDFWc5AgMBAAECggEACIU4/hG+bh97BD7JriFhfDDT6bg7g+pCs/hsAlxQ42jv\nOH7pyWuHJXGf5Cwx31usZAq4fcrgYnVpnyl8odIL628y9AjdI66wMuWhZnBFGJgK\nRhHcZWjW8nlXf0lBjwwFe4edzbn1AuWT5fYZ2HWDW2mthY/e8sUwqWPcWsjdifhz\nNR7V+Ia47McKXYgEKjyEObSP1NUOW24zH0DgxS52YPMwa1FoHn6+9Pr8P3TsTSO6\nh6f8tnd81DGl1UH4F5Bj/MHsQXyAMJbu44S4+rZ4Qlk+5xPp9hfCNpxWaHLIkJCg\nYXnC8UAjjyXiqyK0U0RjJf8TS1FxUI4iPepLNqp/pQKBgQDTicZnWFXmCFTnycWp\n66P3Yx0yvlKdUdfnoD/n9NdmUA3TZUlEVfb0IOm7ZFubF/zDTH87XrRiD/NVDbr8\n6bdhA1DXzraxhbfD36Hca6K74Ba4aYJsSWWwI0hL3FDSsv8c7qAIaUF2iwuHb7Y0\nRDcvZqowtQobcQC8cHLc/bI/ZwKBgQC6fMeGaU+lP6jhp9Nb/3Gz5Z1zzCu34IOo\nlgpTNZsowRKYLtjHifrEFi3XRxPKz5thMuJFniof5U4WoMYtRXy+PbgySvBpCia2\nXty05XssnLLMvLpYU5sbQvmOTe20zaIzLohRvvmqrydYIKu62NTubNeuD1L+Zr0q\nz1P5/wUgXwKBgQCW9MrRFQi3j1qHzkVwbOglsmUzwP3TpoQclw8DyIWuTZKQOMeA\nLJh+vr4NLCDzHLsT45MoGv0+vYM4PwQhV+e1I1idqLZXGMV60iv/0A/hYpjUIPch\nr38RoxwEhsRml7XWP7OUTQiaP7+Kdv3fbo6zFOB+wbLkwk90KgrOCX0aIQKBgFeK\n7esmErJjMPdFXk3om0q09nX+mWNHLOb+EDjBiGXYRM9V5oO9PQ/BzaEqh5sEXE+D\noH7H4cR5U3AB5yYnYYi41ngdf7//eO7Rl1AADhOCN9kum1eNX9mrVhU8deMTSRo3\ntNyTBwbeFF0lcRhUY5jNVW4rWW19cz3ed/B6i8CHAoGBAJ/l5rkV74Z5hg6BWNfQ\nYAg/4PLZmjnXIy5QdnWc/PYgbhn5+iVUcL9fSofFzJM1rjFnNcs3S90MGeOmfmo4\nM1WtcQFQbsCGt6+G5uEL/nf74mKUGpOqEM/XSkZ3inweWiDk3LK3iYfXCMBFouIr\n80IlzI1yMf7MVmWn3e1zPjCA\n-----END PRIVATE KEY-----\n",
-  "client_email": "eveai-349@eveai-420711.iam.gserviceaccount.com",
-  "client_id": "109927035346319712442",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/eveai-349%40eveai-420711.iam.gserviceaccount.com",
-  "universe_domain": "googleapis.com"
-}

config/logging_config.py

@@ -12,7 +12,12 @@ env = os.environ.get('FLASK_ENV', 'development')
 class CustomLogRecord(logging.LogRecord):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')  # Set default component value here
+        self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')
+
+    def __setattr__(self, name, value):
+        if name not in {'event_type', 'tenant_id', 'trace_id', 'span_id', 'span_name', 'parent_span_id',
+                        'document_version_id', 'chat_session_id', 'interaction_id', 'environment'}:
+            super().__setattr__(name, value)
 
 
 def custom_log_record_factory(*args, **kwargs):
@@ -108,6 +113,14 @@ LOGGING = {
             'backupCount': 10,
             'formatter': 'standard',
         },
+        'file_business_events': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'logs/business_events.log',
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
+            'backupCount': 10,
+            'formatter': 'standard',
+        },
         'console': {
             'class': 'logging.StreamHandler',
             'level': 'DEBUG',
@@ -184,6 +197,11 @@ LOGGING = {
             'level': 'DEBUG',
             'propagate': False
         },
+        'business_events': {
+            'handlers': ['file_business_events', 'graylog'],
+            'level': 'DEBUG',
+            'propagate': False
+        },
         '': {  # root logger
             'handlers': ['console'],
             'level': 'WARNING',  # Set higher level for root to minimize noise

config/model_config.py

@@ -0,0 +1,41 @@
+MODEL_CONFIG = {
+    "openai": {
+        "gpt-4o": {
+            "tool_calling_supported": True,
+            "processing_chunk_size": 10000,
+            "processing_chunk_overlap": 200,
+            "processing_min_chunk_size": 8000,
+            "processing_max_chunk_size": 12000,
+            "prompt_templates": [
+                "summary", "rag", "history", "encyclopedia",
+                "transcript", "html_parse", "pdf_parse"
+            ]
+        },
+        "gpt-4o-mini": {
+            "tool_calling_supported": True,
+            "processing_chunk_size": 10000,
+            "processing_chunk_overlap": 200,
+            "processing_min_chunk_size": 8000,
+            "processing_max_chunk_size": 12000,
+            "prompt_templates": [
+                "summary", "rag", "history", "encyclopedia",
+                "transcript", "html_parse", "pdf_parse"
+            ]
+        },
+        # Add other OpenAI models here
+    },
+    "anthropic": {
+        "claude-3-5-sonnet": {
+            "tool_calling_supported": True,
+            "processing_chunk_size": 10000,
+            "processing_chunk_overlap": 200,
+            "processing_min_chunk_size": 8000,
+            "processing_max_chunk_size": 12000,
+            "prompt_templates": [
+                "summary", "rag", "history", "encyclopedia",
+                "transcript", "html_parse", "pdf_parse"
+            ]
+        },
+        # Add other Anthropic models here
+    },
+}

config/prompts/openai/gpt-4o.yaml

@@ -65,11 +65,13 @@ encyclopedia: |
 
 transcript: |
   You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
+  You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk.
+
 
   # Best practices and steps are:
   - Respect wordings and language(s) used in the transcription. Main language is {language}.
   - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.    
-  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part.
+  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences.
   - annotate the text to identify these logical parts using headings in {language}.
   - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.
 
@@ -77,4 +79,6 @@ transcript: |
 
   The transcript is between triple backquotes.
 
+  $$${previous_part}$$$
+
   ```{transcript}```

docker/build_and_push_eveai.sh

@@ -141,7 +141,7 @@ if [ $# -eq 0 ]; then
     SERVICES=()
     while IFS= read -r line; do
         SERVICES+=("$line")
-    done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)')
+    done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_|flower)')
 else
     SERVICES=("$@")
 fi
@@ -158,7 +158,7 @@ docker buildx use eveai_builder
 
 # Loop through services
 for SERVICE in "${SERVICES[@]}"; do
-    if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* ]]; then
+    if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* || "$SERVICE" == "flower" ]]; then
         if process_service "$SERVICE"; then
             echo "Successfully processed $SERVICE"
         else

docker/compose_dev.yaml

@@ -22,6 +22,8 @@ x-common-variables: &common-variables
   MAIL_PASSWORD: '$$6xsWGbNtx$$CFMQZqc*'
   MAIL_SERVER: mail.flow-it.net
   MAIL_PORT: 465
+  REDIS_URL: redis
+  REDIS_PORT: '6379'
   OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7'
   GROQ_API_KEY: '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'
   ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2'
@@ -32,6 +34,7 @@ x-common-variables: &common-variables
   MINIO_ACCESS_KEY: minioadmin
   MINIO_SECRET_KEY: minioadmin
   NGINX_SERVER_NAME: 'localhost http://macstudio.ask-eve-ai-local.com/'
+  LANGCHAIN_API_KEY: "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"
 
 
 networks:
@@ -96,12 +99,11 @@ services:
        minio:
          condition: service_healthy
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-#    entrypoint: ["scripts/entrypoint.sh"]
-#    command: ["scripts/start_eveai_app.sh"]
+      test: ["CMD", "curl", "-f", "http://localhost:5001/healthz/ready"]
+      interval: 30s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
     networks:
       - eveai-network
 
@@ -113,8 +115,6 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_workers
@@ -132,13 +132,6 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
     networks:
       - eveai-network
 
@@ -168,12 +161,11 @@ services:
       redis:
         condition: service_healthy
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ]  # Adjust based on your health endpoint
-      interval: 10s
-      timeout: 5s
-      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
+      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
+      interval: 30s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
     networks:
       - eveai-network
 
@@ -185,8 +177,6 @@ services:
       platforms:
         - linux/amd64
         - linux/arm64
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_chat_workers
@@ -202,13 +192,6 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
     networks:
       - eveai-network
 
@@ -240,12 +223,11 @@ services:
       minio:
         condition: service_healthy
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5003/health" ]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    #    entrypoint: ["scripts/entrypoint.sh"]
-    #    command: ["scripts/start_eveai_api.sh"]
+      test: [ "CMD", "curl", "-f", "http://localhost:5003/healthz/ready" ]
+      interval: 30s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
     networks:
       - eveai-network
 
@@ -285,6 +267,22 @@ services:
     networks:
       - eveai-network
 
+  flower:
+    image: josakola/flower:latest
+    build:
+      context: ..
+      dockerfile: ./docker/flower/Dockerfile
+    environment:
+      <<: *common-variables
+    volumes:
+      - ../scripts:/app/scripts
+    ports:
+      - "5555:5555"
+    depends_on:
+      - redis
+    networks:
+      - eveai-network
+
   minio:
     image: minio/minio
     ports:

docker/compose_stackhero.yaml

@@ -21,11 +21,13 @@ x-common-variables: &common-variables
   MAIL_USERNAME: 'evie_admin@askeveai.com'
   MAIL_PASSWORD: 's5D%R#y^v!s&6Z^i0k&'
   MAIL_SERVER: mail.askeveai.com
-  MAIL_PORT: 465
+  MAIL_PORT: '465'
   REDIS_USER: eveai
   REDIS_PASS: 'jHliZwGD36sONgbm0fc6SOpzLbknqq4RNF8K'
   REDIS_URL: 8bciqc.stackhero-network.com
   REDIS_PORT: '9961'
+  FLOWER_USER: 'Felucia'
+  FLOWER_PASSWORD: 'Jungles'
   OPENAI_API_KEY: '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'
   GROQ_API_KEY: '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'
   ANTHROPIC_API_KEY: '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'
@@ -38,6 +40,7 @@ x-common-variables: &common-variables
   MINIO_ACCESS_KEY: 04JKmQln8PQpyTmMiCPc
   MINIO_SECRET_KEY: 2PEZAD1nlpAmOyDV0TUTuJTQw1qVuYLF3A7GMs0D
   NGINX_SERVER_NAME: 'evie.askeveai.com mxz536.stackhero-network.com'
+  LANGCHAIN_API_KEY: "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"
 
 networks:
   eveai-network:
@@ -53,10 +56,6 @@ services:
     environment:
       <<: *common-variables
     volumes:
-#      - ../nginx:/etc/nginx
-#      - ../nginx/sites-enabled:/etc/nginx/sites-enabled
-#      - ../nginx/static:/etc/nginx/static
-#      - ../nginx/public:/etc/nginx/public
       - eveai_logs:/var/log/nginx
     labels:
       - "traefik.enable=true"
@@ -81,7 +80,7 @@ services:
     volumes:
       - eveai_logs:/app/logs
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
+      test: ["CMD", "curl", "-f", "http://localhost:5001/healthz/ready"]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -91,18 +90,11 @@ services:
   eveai_workers:
     platform: linux/amd64
     image: josakola/eveai_workers:latest
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_workers
     volumes:
       - eveai_logs:/app/logs
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
     networks:
       - eveai-network
 
@@ -117,7 +109,7 @@ services:
     volumes:
       - eveai_logs:/app/logs
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ]  # Adjust based on your health endpoint
+      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
       interval: 10s
       timeout: 5s
       retries: 5
@@ -127,18 +119,11 @@ services:
   eveai_chat_workers:
     platform: linux/amd64
     image: josakola/eveai_chat_workers:latest
-#    ports:
-#      - 5001:5001
     environment:
       <<: *common-variables
       COMPONENT_NAME: eveai_chat_workers
     volumes:
       - eveai_logs:/app/logs
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
     networks:
       - eveai-network
 
@@ -153,20 +138,23 @@ services:
     volumes:
       - eveai_logs:/app/logs
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:5003/healthz/ready" ]
       interval: 10s
       timeout: 5s
       retries: 5
     networks:
       - eveai-network
 
+  flower:
+    image: josakola/flower:latest
+    environment:
+      <<: *common-variables
+    ports:
+      - "5555:5555"
+    networks:
+      - eveai-network
+
 volumes:
   eveai_logs:
-#  miniAre theo_data:
-#  db-data:
-#  redis-data:
-#  tenant-files:
-#secrets:
-#  db-password:
-#    file: ./db/password.txt
+
 

docker/eveai_api/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     gcc \
     postgresql-client \
+    curl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

docker/eveai_app/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     gcc \
     postgresql-client \
+    curl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

docker/eveai_chat/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     gcc \
     postgresql-client \
+    curl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

docker/flower/Dockerfile

@@ -0,0 +1,34 @@
+ARG PYTHON_VERSION=3.12.3
+FROM python:${PYTHON_VERSION}-slim as base
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/bin/bash" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    gcc \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . /app
+COPY scripts/start_flower.sh /app/start_flower.sh
+RUN chmod a+x /app/start_flower.sh
+
+USER appuser
+
+CMD ["/app/start_flower.sh"]

eveai_api/__init__.py

@@ -39,9 +39,12 @@ def create_app(config_file=None):
     # Register Necessary Extensions
     register_extensions(app)
 
-    # register Blueprints
+    # register Namespaces
     register_namespaces(api_rest)
 
+    # Register Blueprints
+    register_blueprints(app)
+
     # Error handler for the API
     @app.errorhandler(EveAIException)
     def handle_eveai_exception(error):
@@ -73,20 +76,24 @@ def create_app(config_file=None):
             app.logger.debug('Token request detected, skipping JWT verification')
             return
 
-        try:
-            verify_jwt_in_request(optional=True)
-            tenant_id = get_jwt_identity()
-            app.logger.debug(f'Tenant ID from JWT: {tenant_id}')
+        # Check if this a health check request
+        if request.path.startswith('/_healthz') or request.path.startswith('/healthz'):
+            app.logger.debug('Health check request detected, skipping JWT verification')
+        else:
+            try:
+                verify_jwt_in_request(optional=True)
+                tenant_id = get_jwt_identity()
+                app.logger.debug(f'Tenant ID from JWT: {tenant_id}')
 
-            if tenant_id:
-                Database(tenant_id).switch_schema()
-                app.logger.debug(f'Switched to schema for tenant {tenant_id}')
-            else:
-                app.logger.debug('No tenant ID found in JWT')
-        except Exception as e:
-            app.logger.error(f'Error in before_request: {str(e)}')
-            # Don't raise the exception here, let the request continue
-            # The appropriate error handling will be done in the specific endpoints
+                if tenant_id:
+                    Database(tenant_id).switch_schema()
+                    app.logger.debug(f'Switched to schema for tenant {tenant_id}')
+                else:
+                    app.logger.debug('No tenant ID found in JWT')
+            except Exception as e:
+                app.logger.error(f'Error in before_request: {str(e)}')
+                # Don't raise the exception here, let the request continue
+                # The appropriate error handling will be done in the specific endpoints
 
     return app
 
@@ -102,3 +109,9 @@ def register_extensions(app):
 def register_namespaces(app):
     api_rest.add_namespace(document_ns, path='/api/v1/documents')
     api_rest.add_namespace(auth_ns, path='/api/v1/auth')
+
+
+def register_blueprints(app):
+    from .views.healthz_views import healthz_bp
+    app.register_blueprint(healthz_bp)
+

eveai_api/views/healthz_views.py

@@ -0,0 +1,82 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+# Define Prometheus metrics
+api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint'])
+api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        # "celery": check_celery(),
+        "minio": check_minio(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('tasks.ping', queue='embeddings')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+def check_minio():
+    try:
+        # List buckets to check if MinIO is accessible
+        minio_client.list_buckets()
+        return True
+    except Exception as e:
+        current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )

eveai_app/__init__.py

@@ -7,9 +7,11 @@ from werkzeug.middleware.proxy_fix import ProxyFix
 import logging.config
 
 from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session,
-                               minio_client, simple_encryption)
+                               minio_client, simple_encryption, metrics)
 from common.models.user import User, Role, Tenant, TenantDomain
 import common.models.interaction
+import common.models.monitoring
+import common.models.document
 from common.utils.nginx_utils import prefixed_url_for
 from config.logging_config import LOGGING
 from common.utils.security import set_tenant_session_data
@@ -114,10 +116,10 @@ def register_extensions(app):
     csrf.init_app(app)
     login_manager.init_app(app)
     cors.init_app(app)
-    # kms_client.init_app(app)
     simple_encryption.init_app(app)
     session.init_app(app)
     minio_client.init_app(app)
+    metrics.init_app(app)
 
 
 # Register Blueprints
@@ -132,3 +134,7 @@ def register_blueprints(app):
     app.register_blueprint(security_bp)
     from .views.interaction_views import interaction_bp
     app.register_blueprint(interaction_bp)
+    from .views.healthz_views import healthz_bp, init_healtz
+    app.register_blueprint(healthz_bp)
+    init_healtz(app)
+

eveai_app/templates/macros.html

@@ -1,16 +1,16 @@
-{% macro render_field(field, disabled_fields=[], exclude_fields=[]) %}
+{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %}
     {% set disabled = field.name in disabled_fields %}
     {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %}
     {% if field.name not in exclude_fields %}
         {% if field.type == 'BooleanField' %}
             <div class="form-check">
-                {{ field(class="form-check-input", type="checkbox", id="flexSwitchCheckDefault") }}
+                {{ field(class="form-check-input " + class, type="checkbox", id="flexSwitchCheckDefault") }}
                 {{ field.label(class="form-check-label", for="flexSwitchCheckDefault", disabled=disabled) }}
             </div>
         {% else %}
             <div class="form-group">
                 {{ field.label(class="form-label") }}
-                {{ field(class="form-control", disabled=disabled) }}
+                {{ field(class="form-control " + class, disabled=disabled) }}
                 {% if field.errors %}
                     <div class="invalid-feedback">
                         {% for error in field.errors %}

eveai_app/templates/scripts.html

@@ -13,3 +13,5 @@
         <script src="{{url_for('static', filename='assets/js/plugins/anime.min.js')}}"></script>
         <script src="{{url_for('static', filename='assets/js/material-kit-pro.min.js')}}?v=3.0.4 type="text/javascript"></script>
         <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/js/bootstrap.bundle.min.js"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.13/js/select2.min.js"></script>
+

eveai_app/templates/user/select_tenant.html

@@ -1,22 +1,52 @@
 {% extends 'base.html' %}
-{% from "macros.html" import render_selectable_table, render_pagination %}
-
+{% from "macros.html" import render_selectable_table, render_pagination, render_field %}
 {% block title %}Tenant Selection{% endblock %}
-
 {% block content_title %}Select a Tenant{% endblock %}
 {% block content_description %}Select the active tenant for the current session{% endblock %}
-
 {% block content %}
+
+<!-- Filter Form -->
+<form method="POST" action="{{ url_for('user_bp.select_tenant') }}" class="mb-4">
+    {{ filter_form.hidden_tag() }}
+    <div class="row">
+        <div class="col-md-4">
+            {{ render_field(filter_form.types, class="select2") }}
+        </div>
+        <div class="col-md-4">
+            {{ render_field(filter_form.search) }}
+        </div>
+        <div class="col-md-4">
+            {{ filter_form.submit(class="btn btn-primary") }}
+        </div>
+    </div>
+</form>
+
+<!-- Tenant Selection Form -->
 <form method="POST" action="{{ url_for('user_bp.handle_tenant_selection') }}">
-    {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website"], rows=rows, selectable=True, id="tenantsTable") }}
+    {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website", "Type"], rows=rows, selectable=True, id="tenantsTable") }}
     <div class="form-group mt-3">
         <button type="submit" name="action" value="select_tenant" class="btn btn-primary">Set Session Tenant</button>
         <button type="submit" name="action" value="edit_tenant" class="btn btn-secondary">Edit Tenant</button>
     </div>
 </form>
+
 {% endblock %}
 
 {% block content_footer %}
-    {{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{% endblock %}
+
+{% block scripts %}
+<script>
+$(document).ready(function() {
+    $('.select2').select2({
+        placeholder: "Select tenant types",
+        allowClear: true,
+        minimumResultsForSearch: Infinity, // Hides the search box
+        dropdownCssClass: 'select2-dropdown-hidden', // Custom class for dropdown
+        containerCssClass: 'select2-container-hidden' // Custom class for container
+    });
+});
+</script>
 {% endblock %}
 

eveai_app/templates/user/tenant_overview.html

@@ -10,7 +10,7 @@
     <form method="post">
         {{ form.hidden_tag() }}
         <!-- Main Tenant Information -->
-        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages'] %}
+        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages', 'rag_context', 'type'] %}
         {% for field in form %}
             {{ render_included_field(field, disabled_fields=main_fields, include_fields=main_fields) }}
         {% endfor %}

eveai_app/views/document_forms.py

@@ -30,7 +30,6 @@ class AddDocumentForm(FlaskForm):
     user_context = TextAreaField('User Context', validators=[Optional()])
     valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
     user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
-    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
 
     submit = SubmitField('Submit')
 
@@ -38,7 +37,8 @@ class AddDocumentForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
-        self.language.data = session.get('tenant').get('default_language')
+        if not self.language.data:
+            self.language.data = session.get('tenant').get('default_language')
 
 
 class AddURLForm(FlaskForm):
@@ -48,7 +48,6 @@ class AddURLForm(FlaskForm):
     user_context = TextAreaField('User Context', validators=[Optional()])
     valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
     user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
-    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
 
     submit = SubmitField('Submit')
 
@@ -56,7 +55,8 @@ class AddURLForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
-        self.language.data = session.get('tenant').get('default_language')
+        if not self.language.data:
+            self.language.data = session.get('tenant').get('default_language')
 
 
 class AddURLsForm(FlaskForm):
@@ -72,7 +72,8 @@ class AddURLsForm(FlaskForm):
         super().__init__()
         self.language.choices = [(language, language) for language in
                                  session.get('tenant').get('allowed_languages')]
-        self.language.data = session.get('tenant').get('default_language')
+        if not self.language.data:
+            self.language.data = session.get('tenant').get('default_language')
 
 
 class EditDocumentForm(FlaskForm):

eveai_app/views/document_views.py

@@ -56,11 +56,9 @@ def before_request():
 @roles_accepted('Super User', 'Tenant Admin')
 def add_document():
     form = AddDocumentForm()
-    current_app.logger.debug('Adding document')
 
     if form.validate_on_submit():
         try:
-            current_app.logger.debug('Validating file type')
             tenant_id = session['tenant']['id']
             file = form.file.data
             filename = secure_filename(file.filename)
@@ -68,15 +66,15 @@ def add_document():
 
             validate_file_type(extension)
 
+            current_app.logger.debug(f'Language on form: {form.language.data}')
             api_input = {
                 'name': form.name.data,
                 'language': form.language.data,
                 'user_context': form.user_context.data,
                 'valid_from': form.valid_from.data,
                 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
-                'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None
-
             }
+            current_app.logger.debug(f'Creating document stack with input {api_input}')
 
             new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
             task_id = start_embedding_task(tenant_id, new_doc_vers.id)
@@ -113,7 +111,6 @@ def add_url():
                 'user_context': form.user_context.data,
                 'valid_from': form.valid_from.data,
                 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
-                'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None
             }
 
             new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id)

eveai_app/views/healthz_views.py

@@ -0,0 +1,100 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+import time
+
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+# Define Prometheus metrics
+api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint'])
+api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        "minio": check_minio(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('ping', queue='embeddings')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+def check_minio():
+    try:
+        # List buckets to check if MinIO is accessible
+        minio_client.list_buckets()
+        return True
+    except Exception as e:
+        current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
+
+
+# Custom metrics example
+@healthz_bp.before_app_request
+def before_request():
+    request.start_time = time.time()
+    api_request_counter.labels(
+        method=request.method, endpoint=request.endpoint
+    ).inc()
+
+
+@healthz_bp.after_app_request
+def after_request(response):
+    request_duration = time.time() - request.start_time
+    api_request_latency.observe(request_duration)
+    return response
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )

eveai_app/views/user_forms.py

@@ -2,7 +2,7 @@ from flask import current_app
 from flask_wtf import FlaskForm
 from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
                      SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField)
-from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional
+from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError
 import pytz
 
 from common.models.user import Role
@@ -18,6 +18,8 @@ class TenantForm(FlaskForm):
     timezone = SelectField('Timezone', choices=[], validators=[DataRequired()])
     # RAG context
     rag_context = TextAreaField('RAG Context', validators=[Optional()])
+    # Tenant Type
+    type = SelectField('Tenant Type', validators=[Optional()], default='Active')
     # LLM fields
     embedding_model = SelectField('Embedding Model', choices=[], validators=[DataRequired()])
     llm_model = SelectField('Large Language Model', choices=[], validators=[DataRequired()])
@@ -65,6 +67,7 @@ class TenantForm(FlaskForm):
         # Initialize fallback algorithms
         self.fallback_algorithms.choices = \
             [(algorithm, algorithm.lower()) for algorithm in current_app.config['FALLBACK_ALGORITHMS']]
+        self.type.choices = [(t, t) for t in current_app.config['TENANT_TYPES']]
 
 
 class BaseUserForm(FlaskForm):
@@ -107,4 +110,14 @@ class TenantDomainForm(FlaskForm):
     submit = SubmitField('Add Domain')
 
 
+class TenantSelectionForm(FlaskForm):
+    types = SelectMultipleField('Tenant Types', choices=[], validators=[Optional()])
+    search = StringField('Search', validators=[Optional()])
+    submit = SubmitField('Filter')
+
+    def __init__(self, *args, **kwargs):
+        super(TenantSelectionForm, self).__init__(*args, **kwargs)
+        self.types.choices = [(t, t) for t in current_app.config['TENANT_TYPES']]
+
+
 

eveai_app/views/user_views.py

@@ -10,7 +10,7 @@ import ast
 from common.models.user import User, Tenant, Role, TenantDomain
 from common.extensions import db, security, minio_client, simple_encryption
 from common.utils.security_utils import send_confirmation_email, send_reset_email
-from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm
+from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm, TenantSelectionForm
 from common.utils.database import Database
 from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
 from common.utils.simple_encryption import generate_api_key
@@ -129,6 +129,7 @@ def edit_tenant(tenant_id):
             form.html_excluded_classes.data = ', '.join(tenant.html_excluded_classes)
 
     if form.validate_on_submit():
+        current_app.logger.debug(f'Updating tenant {tenant_id}')
         # Populate the tenant with form data
         form.populate_obj(tenant)
         # Then handle the special fields manually
@@ -148,6 +149,7 @@ def edit_tenant(tenant_id):
                 session['tenant'] = tenant.to_dict()
         # return redirect(url_for(f"user/tenant/tenant_id"))
     else:
+        current_app.logger.debug(f'Tenant update failed with errors: {form.errors}')
         form_validation_failed(request, form)
 
     return render_template('user/edit_tenant.html', form=form, tenant_id=tenant_id)
@@ -245,20 +247,29 @@ def edit_user(user_id):
     return render_template('user/edit_user.html', form=form, user_id=user_id)
 
 
-@user_bp.route('/select_tenant')
+@user_bp.route('/select_tenant', methods=['GET', 'POST'])
 @roles_required('Super User')
 def select_tenant():
+    filter_form = TenantSelectionForm(request.form)
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
 
-    query = Tenant.query.order_by(Tenant.name)  # Fetch all tenants from the database
+    query = Tenant.query
 
-    pagination = query.paginate(page=page, per_page=per_page)
+    if filter_form.validate_on_submit():
+        if filter_form.types.data:
+            query = query.filter(Tenant.type.in_(filter_form.types.data))
+        if filter_form.search.data:
+            search = f"%{filter_form.search.data}%"
+            query = query.filter(Tenant.name.ilike(search))
+
+    query = query.order_by(Tenant.name)
+    pagination = query.paginate(page=page, per_page=per_page, error_out=False)
     tenants = pagination.items
 
-    rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', '')])
+    rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', ''), ('type', '')])
 
-    return render_template('user/select_tenant.html', rows=rows, pagination=pagination)
+    return render_template('user/select_tenant.html', rows=rows, pagination=pagination, filter_form=filter_form)
 
 
 @user_bp.route('/handle_tenant_selection', methods=['POST'])

eveai_chat/__init__.py

@@ -3,7 +3,7 @@ import logging.config
 from flask import Flask, jsonify
 import os
 
-from common.extensions import db, socketio, jwt, cors, session, simple_encryption
+from common.extensions import db, socketio, jwt, cors, session, simple_encryption, metrics
 from config.logging_config import LOGGING
 from eveai_chat.socket_handlers import chat_handler
 from common.utils.cors_utils import create_cors_after_request
@@ -32,17 +32,6 @@ def create_app(config_file=None):
     app.celery = make_celery(app.name, app.config)
     init_celery(app.celery, app)
 
-    # Register Blueprints
-    # register_blueprints(app)
-
-    @app.route('/ping')
-    def ping():
-        return 'pong'
-
-    @app.route('/health', methods=['GET'])
-    def health():
-        return jsonify({'status': 'ok'}), 200
-
     app.logger.info("EveAI Chat Server Started Successfully")
     app.logger.info("-------------------------------------------------------------------------------------------------")
     return app
@@ -61,8 +50,8 @@ def register_extensions(app):
                       ping_interval=app.config.get('SOCKETIO_PING_INTERVAL'),
                       )
     jwt.init_app(app)
-    # kms_client.init_app(app)
     simple_encryption.init_app(app)
+    metrics.init_app(app)
 
     # Cors setup
     cors.init_app(app, resources={r"/chat/*": {"origins": "*"}})
@@ -72,5 +61,5 @@ def register_extensions(app):
 
 
 def register_blueprints(app):
-    from .views.chat_views import chat_bp
-    app.register_blueprint(chat_bp)
+    from views.healthz_views import healthz_bp
+    app.register_blueprint(healthz_bp)

eveai_chat/socket_handlers/chat_handler.py

@@ -1,10 +1,13 @@
 import uuid
+from functools import wraps
 
 from flask_jwt_extended import create_access_token, get_jwt_identity, verify_jwt_in_request, decode_token
 from flask_socketio import emit, disconnect, join_room, leave_room
 from flask import current_app, request, session
 from sqlalchemy.exc import SQLAlchemyError
 from datetime import datetime, timedelta
+from prometheus_client import Counter, Histogram
+from time import time
 
 from common.extensions import socketio, db, simple_encryption
 from common.models.user import Tenant
@@ -12,8 +15,27 @@ from common.models.interaction import Interaction
 from common.utils.celery_utils import current_celery
 from common.utils.database import Database
 
+# Define custom metrics
+socketio_message_counter = Counter('socketio_message_count', 'Count of SocketIO messages', ['event_type'])
+socketio_message_latency = Histogram('socketio_message_latency_seconds', 'Latency of SocketIO message processing', ['event_type'])
+
+
+# Decorator to measure SocketIO events
+def track_socketio_event(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        event_type = func.__name__
+        socketio_message_counter.labels(event_type=event_type).inc()
+        start_time = time()
+        result = func(*args, **kwargs)
+        latency = time() - start_time
+        socketio_message_latency.labels(event_type=event_type).observe(latency)
+        return result
+    return wrapper
+
 
 @socketio.on('connect')
+@track_socketio_event
 def handle_connect():
     try:
         current_app.logger.debug(f'SocketIO: Connection handling started using {request.args}')
@@ -58,6 +80,7 @@ def handle_connect():
 
 
 @socketio.on('disconnect')
+@track_socketio_event
 def handle_disconnect():
     room = session.get('room')
     if room:

eveai_chat/views/chat_views.py

@@ -1,77 +0,0 @@
-from datetime import datetime as dt, timezone as tz
-from flask import request, redirect, url_for, render_template, Blueprint, session, current_app, jsonify
-from flask_security import hash_password, roles_required, roles_accepted
-from sqlalchemy.exc import SQLAlchemyError
-from flask_jwt_extended import create_access_token, jwt_required, get_jwt_identity
-from flask_socketio import emit, join_room, leave_room
-import ast
-
-
-from common.models.user import User, Tenant
-from common.models.interaction import ChatSession, Interaction, InteractionEmbedding
-from common.models.document import Embedding
-from common.extensions import db, socketio, kms_client
-from common.utils.database import Database
-
-chat_bp = Blueprint('chat_bp', __name__, url_prefix='/chat')
-
-
-@chat_bp.route('/register_client', methods=['POST'])
-def register_client():
-    tenant_id = request.json.get('tenant_id')
-    api_key = request.json.get('api_key')
-
-    # Validate tenant_id and api_key here (e.g., check against the database)
-    if validate_tenant(tenant_id, api_key):
-        access_token = create_access_token(identity={'tenant_id': tenant_id, 'api_key': api_key})
-        current_app.logger.debug(f'Tenant Registration: Tenant {tenant_id} registered successfully')
-        return jsonify({'token': access_token}), 200
-    else:
-        current_app.logger.debug(f'Tenant Registration: Invalid tenant_id ({tenant_id}) or api_key ({api_key})')
-        return jsonify({'message': 'Invalid credentials'}), 401
-
-
-@socketio.on('connect', namespace='/chat')
-@jwt_required()
-def handle_connect():
-    current_tenant = get_jwt_identity()
-    current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} connected')
-
-
-@socketio.on('message', namespace='/chat')
-@jwt_required()
-def handle_message(data):
-    current_tenant = get_jwt_identity()
-    current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} sent a message: {data}')
-    # Store interaction in the database
-    emit('response', {'data': 'Message received'}, broadcast=True)
-
-
-def validate_tenant(tenant_id, api_key):
-    tenant = Tenant.query.get_or_404(tenant_id)
-    encrypted_api_key = ast.literal_eval(tenant.encrypted_chat_api_key)
-
-    decrypted_api_key = kms_client.decrypt_api_key(encrypted_api_key)
-
-    return decrypted_api_key == api_key
-
-
-
-# @chat_bp.route('/', methods=['GET', 'POST'])
-# def chat():
-#     return render_template('chat.html')
-#
-#
-# @chat.record_once
-# def on_register(state):
-#     # TODO: write initialisation code when the blueprint is registered (only once)
-#     # socketio.init_app(state.app)
-#     pass
-#
-#
-# @socketio.on('message', namespace='/chat')
-# def handle_message(message):
-#     # TODO: write message handling code to actually realise chat
-#     # print('Received message:', message)
-#     # socketio.emit('response', {'data': message}, namespace='/chat')
-#     pass

eveai_chat/views/healthz_views.py

@@ -0,0 +1,70 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+from eveai_chat.socket_handlers.chat_handler import socketio_message_counter, socketio_message_latency
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('ping', queue='llm_interactions')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return metrics.generate_latest()
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )
+    # Register SocketIO metrics with Prometheus
+    metrics.register(socketio_message_counter)
+    metrics.register(socketio_message_latency)

eveai_chat_workers/tasks.py

@@ -22,12 +22,23 @@ from common.models.interaction import ChatSession, Interaction, InteractionEmbed
 from common.extensions import db
 from common.utils.celery_utils import current_celery
 from common.utils.model_utils import select_model_variables, create_language_template, replace_variable_in_template
-from common.langchain.EveAIRetriever import EveAIRetriever
-from common.langchain.EveAIHistoryRetriever import EveAIHistoryRetriever
+from common.langchain.eveai_retriever import EveAIRetriever
+from common.langchain.eveai_history_retriever import EveAIHistoryRetriever
+from common.utils.business_event import BusinessEvent
+from common.utils.business_event_context import current_event
+
+
+# Healthcheck task
+@current_celery.task(name='ping', queue='llm_interactions')
+def ping():
+    return 'pong'
 
 
 def detail_question(question, language, model_variables, session_id):
-    retriever = EveAIHistoryRetriever(model_variables, session_id)
+    current_app.logger.debug(f'Detail question: {question}')
+    current_app.logger.debug(f'model_varialbes: {model_variables}')
+    current_app.logger.debug(f'session_id: {session_id}')
+    retriever = EveAIHistoryRetriever(model_variables=model_variables, session_id=session_id)
     llm = model_variables['llm']
     template = model_variables['history_template']
     language_template = create_language_template(template, language)
@@ -56,53 +67,56 @@ def ask_question(tenant_id, question, language, session_id, user_timezone, room)
     'interaction_id': 'interaction_id_value'
     }
     """
-    current_app.logger.info(f'ask_question: Received question for tenant {tenant_id}: {question}. Processing...')
+    with BusinessEvent("Ask Question", tenant_id=tenant_id, chat_session_id=session_id):
+        current_app.logger.info(f'ask_question: Received question for tenant {tenant_id}: {question}. Processing...')
 
-    try:
-        # Retrieve the tenant
-        tenant = Tenant.query.get(tenant_id)
-        if not tenant:
-            raise Exception(f'Tenant {tenant_id} not found.')
+        try:
+            # Retrieve the tenant
+            tenant = Tenant.query.get(tenant_id)
+            if not tenant:
+                raise Exception(f'Tenant {tenant_id} not found.')
 
-        # Ensure we are working in the correct database schema
-        Database(tenant_id).switch_schema()
+            # Ensure we are working in the correct database schema
+            Database(tenant_id).switch_schema()
 
-        # Ensure we have a session to story history
-        chat_session = ChatSession.query.filter_by(session_id=session_id).first()
-        if not chat_session:
-            try:
-                chat_session = ChatSession()
-                chat_session.session_id = session_id
-                chat_session.session_start = dt.now(tz.utc)
-                chat_session.timezone = user_timezone
-                db.session.add(chat_session)
-                db.session.commit()
-            except SQLAlchemyError as e:
-                current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}')
-                raise
+            # Ensure we have a session to story history
+            chat_session = ChatSession.query.filter_by(session_id=session_id).first()
+            if not chat_session:
+                try:
+                    chat_session = ChatSession()
+                    chat_session.session_id = session_id
+                    chat_session.session_start = dt.now(tz.utc)
+                    chat_session.timezone = user_timezone
+                    db.session.add(chat_session)
+                    db.session.commit()
+                except SQLAlchemyError as e:
+                    current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}')
+                    raise
 
-        if tenant.rag_tuning:
-            current_app.rag_tuning_logger.debug(f'Received question for tenant {tenant_id}:\n{question}. Processing...')
-            current_app.rag_tuning_logger.debug(f'Tenant Information: \n{tenant.to_dict()}')
-            current_app.rag_tuning_logger.debug(f'===================================================================')
-            current_app.rag_tuning_logger.debug(f'===================================================================')
+            if tenant.rag_tuning:
+                current_app.rag_tuning_logger.debug(f'Received question for tenant {tenant_id}:\n{question}. Processing...')
+                current_app.rag_tuning_logger.debug(f'Tenant Information: \n{tenant.to_dict()}')
+                current_app.rag_tuning_logger.debug(f'===================================================================')
+                current_app.rag_tuning_logger.debug(f'===================================================================')
 
-        result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session)
-        result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
-        result['interaction_id'] = interaction.id
-        result['room'] = room  # Include the room in the result
-
-        if result['insufficient_info']:
-            if 'LLM' in tenant.fallback_algorithms:
-                result, interaction = answer_using_llm(question, language, tenant, chat_session)
-                result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name']
+            with current_event.create_span("RAG Answer"):
+                result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session)
+                result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
                 result['interaction_id'] = interaction.id
                 result['room'] = room  # Include the room in the result
 
-        return result
-    except Exception as e:
-        current_app.logger.error(f'ask_question: Error processing question: {e}')
-        raise
+            if result['insufficient_info']:
+                if 'LLM' in tenant.fallback_algorithms:
+                    with current_event.create_span("Fallback Algorithm LLM"):
+                        result, interaction = answer_using_llm(question, language, tenant, chat_session)
+                        result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name']
+                        result['interaction_id'] = interaction.id
+                        result['room'] = room  # Include the room in the result
+
+            return result
+        except Exception as e:
+            current_app.logger.error(f'ask_question: Error processing question: {e}')
+            raise
 
 
 def answer_using_tenant_rag(question, language, tenant, chat_session):
@@ -122,92 +136,94 @@ def answer_using_tenant_rag(question, language, tenant, chat_session):
     # Langchain debugging if required
     # set_debug(True)
 
-    detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
-    current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
-    if tenant.rag_tuning:
-        current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.')
-        current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
-    new_interaction.detailed_question = detailed_question
-    new_interaction.detailed_question_at = dt.now(tz.utc)
-
-    retriever = EveAIRetriever(model_variables, tenant_info)
-    llm = model_variables['llm']
-    template = model_variables['rag_template']
-    language_template = create_language_template(template, language)
-    full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
-    rag_prompt = ChatPromptTemplate.from_template(full_template)
-    setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()})
-    if tenant.rag_tuning:
-        current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.')
-        current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
-
-    new_interaction_embeddings = []
-    if not model_variables['cited_answer_cls']:  # The model doesn't support structured feedback
-        output_parser = StrOutputParser()
-
-        chain = setup_and_retrieval | rag_prompt | llm | output_parser
-
-        # Invoke the chain with the actual question
-        answer = chain.invoke(detailed_question)
-        new_interaction.answer = answer
-        result = {
-            'answer': answer,
-            'citations': [],
-            'insufficient_info': False
-        }
-
-    else:  # The model supports structured feedback
-        structured_llm = llm.with_structured_output(model_variables['cited_answer_cls'])
-
-        chain = setup_and_retrieval | rag_prompt | structured_llm
-
-        result = chain.invoke(detailed_question).dict()
-        current_app.logger.debug(f'ask_question: result answer: {result['answer']}')
-        current_app.logger.debug(f'ask_question: result citations: {result["citations"]}')
-        current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
+    with current_event.create_span("Detail Question"):
+        detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
+        current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
         if tenant.rag_tuning:
-            current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}')
-            current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}')
-            current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
+            current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.')
             current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
-        new_interaction.answer = result['answer']
+        new_interaction.detailed_question = detailed_question
+        new_interaction.detailed_question_at = dt.now(tz.utc)
 
-        # Filter out the existing Embedding IDs
-        given_embedding_ids = [int(emb_id) for emb_id in result['citations']]
-        embeddings = (
-            db.session.query(Embedding)
-            .filter(Embedding.id.in_(given_embedding_ids))
-            .all()
-        )
-        existing_embedding_ids = [emb.id for emb in embeddings]
-        urls = list(set(emb.document_version.url for emb in embeddings))
+    with current_event.create_span("Generate Answer using RAG"):
+        retriever = EveAIRetriever(model_variables, tenant_info)
+        llm = model_variables['llm']
+        template = model_variables['rag_template']
+        language_template = create_language_template(template, language)
+        full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
+        rag_prompt = ChatPromptTemplate.from_template(full_template)
+        setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()})
         if tenant.rag_tuning:
-            current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n')
-            current_app.rag_tuning_logger.debug(f'{urls}')
+            current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.')
             current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
 
-        for emb_id in existing_embedding_ids:
-            new_interaction_embedding = InteractionEmbedding(embedding_id=emb_id)
-            new_interaction_embedding.interaction = new_interaction
-            new_interaction_embeddings.append(new_interaction_embedding)
+        new_interaction_embeddings = []
+        if not model_variables['cited_answer_cls']:  # The model doesn't support structured feedback
+            output_parser = StrOutputParser()
 
-        result['citations'] = urls
+            chain = setup_and_retrieval | rag_prompt | llm | output_parser
 
-    # Disable langchain debugging if set above.
-    # set_debug(False)
+            # Invoke the chain with the actual question
+            answer = chain.invoke(detailed_question)
+            new_interaction.answer = answer
+            result = {
+                'answer': answer,
+                'citations': [],
+                'insufficient_info': False
+            }
 
-    new_interaction.answer_at = dt.now(tz.utc)
-    chat_session.session_end = dt.now(tz.utc)
+        else:  # The model supports structured feedback
+            structured_llm = llm.with_structured_output(model_variables['cited_answer_cls'])
 
-    try:
-        db.session.add(chat_session)
-        db.session.add(new_interaction)
-        db.session.add_all(new_interaction_embeddings)
-        db.session.commit()
-        return result, new_interaction
-    except SQLAlchemyError as e:
-        current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
-        raise
+            chain = setup_and_retrieval | rag_prompt | structured_llm
+
+            result = chain.invoke(detailed_question).dict()
+            current_app.logger.debug(f'ask_question: result answer: {result['answer']}')
+            current_app.logger.debug(f'ask_question: result citations: {result["citations"]}')
+            current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
+            if tenant.rag_tuning:
+                current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}')
+                current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}')
+                current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
+                current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
+            new_interaction.answer = result['answer']
+
+            # Filter out the existing Embedding IDs
+            given_embedding_ids = [int(emb_id) for emb_id in result['citations']]
+            embeddings = (
+                db.session.query(Embedding)
+                .filter(Embedding.id.in_(given_embedding_ids))
+                .all()
+            )
+            existing_embedding_ids = [emb.id for emb in embeddings]
+            urls = list(set(emb.document_version.url for emb in embeddings))
+            if tenant.rag_tuning:
+                current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n')
+                current_app.rag_tuning_logger.debug(f'{urls}')
+                current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
+
+            for emb_id in existing_embedding_ids:
+                new_interaction_embedding = InteractionEmbedding(embedding_id=emb_id)
+                new_interaction_embedding.interaction = new_interaction
+                new_interaction_embeddings.append(new_interaction_embedding)
+
+            result['citations'] = urls
+
+        # Disable langchain debugging if set above.
+        # set_debug(False)
+
+        new_interaction.answer_at = dt.now(tz.utc)
+        chat_session.session_end = dt.now(tz.utc)
+
+        try:
+            db.session.add(chat_session)
+            db.session.add(new_interaction)
+            db.session.add_all(new_interaction_embeddings)
+            db.session.commit()
+            return result, new_interaction
+        except SQLAlchemyError as e:
+            current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
+            raise
 
 
 def answer_using_llm(question, language, tenant, chat_session):
@@ -227,47 +243,49 @@ def answer_using_llm(question, language, tenant, chat_session):
     # Langchain debugging if required
     # set_debug(True)
 
-    detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
-    current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
-    new_interaction.detailed_question = detailed_question
-    new_interaction.detailed_question_at = dt.now(tz.utc)
+    with current_event.create_span("Detail Question"):
+        detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
+        current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
+        new_interaction.detailed_question = detailed_question
+        new_interaction.detailed_question_at = dt.now(tz.utc)
 
-    retriever = EveAIRetriever(model_variables, tenant_info)
-    llm = model_variables['llm_no_rag']
-    template = model_variables['encyclopedia_template']
-    language_template = create_language_template(template, language)
-    rag_prompt = ChatPromptTemplate.from_template(language_template)
-    setup = RunnablePassthrough()
-    output_parser = StrOutputParser()
+    with current_event.create_span("Detail Answer using LLM"):
+        retriever = EveAIRetriever(model_variables, tenant_info)
+        llm = model_variables['llm_no_rag']
+        template = model_variables['encyclopedia_template']
+        language_template = create_language_template(template, language)
+        rag_prompt = ChatPromptTemplate.from_template(language_template)
+        setup = RunnablePassthrough()
+        output_parser = StrOutputParser()
 
-    new_interaction_embeddings = []
+        new_interaction_embeddings = []
 
-    chain = setup | rag_prompt | llm | output_parser
-    input_question = {"question": detailed_question}
+        chain = setup | rag_prompt | llm | output_parser
+        input_question = {"question": detailed_question}
 
-    # Invoke the chain with the actual question
-    answer = chain.invoke(input_question)
-    new_interaction.answer = answer
-    result = {
-        'answer': answer,
-        'citations': [],
-        'insufficient_info': False
-    }
+        # Invoke the chain with the actual question
+        answer = chain.invoke(input_question)
+        new_interaction.answer = answer
+        result = {
+            'answer': answer,
+            'citations': [],
+            'insufficient_info': False
+        }
 
-    # Disable langchain debugging if set above.
-    # set_debug(False)
+        # Disable langchain debugging if set above.
+        # set_debug(False)
 
-    new_interaction.answer_at = dt.now(tz.utc)
-    chat_session.session_end = dt.now(tz.utc)
+        new_interaction.answer_at = dt.now(tz.utc)
+        chat_session.session_end = dt.now(tz.utc)
 
-    try:
-        db.session.add(chat_session)
-        db.session.add(new_interaction)
-        db.session.commit()
-        return result, new_interaction
-    except SQLAlchemyError as e:
-        current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
-        raise
+        try:
+            db.session.add(chat_session)
+            db.session.add(new_interaction)
+            db.session.commit()
+            return result, new_interaction
+        except SQLAlchemyError as e:
+            current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
+            raise
 
 
 def tasks_ping():

eveai_workers/Processors/audio_processor.py

@@ -1,45 +1,37 @@
 import io
 import os
+
 from pydub import AudioSegment
 import tempfile
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
-from common.utils.model_utils import create_language_template
-from .processor import Processor
 import subprocess
 
+from .transcription_processor import TranscriptionProcessor
+from common.utils.business_event_context import current_event
 
-class AudioProcessor(Processor):
+
+class AudioProcessor(TranscriptionProcessor):
     def __init__(self, tenant, model_variables, document_version):
         super().__init__(tenant, model_variables, document_version)
         self.transcription_client = model_variables['transcription_client']
         self.transcription_model = model_variables['transcription_model']
         self.ffmpeg_path = 'ffmpeg'
 
+    def _get_transcription(self):
+        file_data = minio_client.download_document_file(
+            self.tenant.id,
+            self.document_version.doc_id,
+            self.document_version.language,
+            self.document_version.id,
+            self.document_version.file_name
+        )
 
-    def process(self):
-        self._log("Starting Audio processing")
-        try:
-            file_data = minio_client.download_document_file(
-                self.tenant.id,
-                self.document_version.doc_id,
-                self.document_version.language,
-                self.document_version.id,
-                self.document_version.file_name
-            )
-
+        with current_event.create_span("Audio Processing"):
             compressed_audio = self._compress_audio(file_data)
+        with current_event.create_span("Transcription Generation"):
             transcription = self._transcribe_audio(compressed_audio)
-            markdown, title = self._generate_markdown_from_transcription(transcription)
 
-            self._save_markdown(markdown)
-            self._log("Finished processing Audio")
-            return markdown, title
-        except Exception as e:
-            self._log(f"Error processing Audio: {str(e)}", level='error')
-            raise
+        return transcription
 
     def _compress_audio(self, audio_data):
         self._log("Compressing audio")
@@ -159,29 +151,3 @@ class AudioProcessor(Processor):
 
         return full_transcription
 
-    def _generate_markdown_from_transcription(self, transcription):
-        self._log("Generating markdown from transcription")
-        llm = self.model_variables['llm']
-        template = self.model_variables['transcript_template']
-        language_template = create_language_template(template, self.document_version.language)
-        transcript_prompt = ChatPromptTemplate.from_template(language_template)
-        setup = RunnablePassthrough()
-        output_parser = StrOutputParser()
-
-        chain = setup | transcript_prompt | llm | output_parser
-
-        input_transcript = {'transcript': transcription}
-        markdown = chain.invoke(input_transcript)
-
-        # Extract title from the markdown
-        title = self._extract_title_from_markdown(markdown)
-
-        return markdown, title
-
-    def _extract_title_from_markdown(self, markdown):
-        # Simple extraction of the first header as the title
-        lines = markdown.split('\n')
-        for line in lines:
-            if line.startswith('# '):
-                return line[2:].strip()
-        return "Untitled Audio Transcription"

eveai_workers/Processors/html_processor.py

@@ -5,6 +5,7 @@ from langchain_core.runnables import RunnablePassthrough
 from common.extensions import db, minio_client
 from common.utils.model_utils import create_language_template
 from .processor import Processor
+from common.utils.business_event_context import current_event
 
 
 class HTMLProcessor(Processor):
@@ -14,6 +15,9 @@ class HTMLProcessor(Processor):
         self.html_end_tags = model_variables['html_end_tags']
         self.html_included_elements = model_variables['html_included_elements']
         self.html_excluded_elements = model_variables['html_excluded_elements']
+        self.chunk_size = model_variables['processing_chunk_size']  # Adjust this based on your LLM's optimal input size
+        self.chunk_overlap = model_variables[
+            'processing_chunk_overlap']  # Adjust for context preservation between chunks
 
     def process(self):
         self._log("Starting HTML processing")
@@ -27,8 +31,10 @@ class HTMLProcessor(Processor):
             )
             html_content = file_data.decode('utf-8')
 
-            extracted_html, title = self._parse_html(html_content)
-            markdown = self._generate_markdown_from_html(extracted_html)
+            with current_event.create_span("HTML Content Extraction"):
+                extracted_html, title = self._parse_html(html_content)
+            with current_event.create_span("Markdown Generation"):
+                markdown = self._generate_markdown_from_html(extracted_html)
 
             self._save_markdown(markdown)
             self._log("Finished processing HTML")
@@ -70,7 +76,7 @@ class HTMLProcessor(Processor):
         chain = setup | parse_prompt | llm | output_parser
 
         soup = BeautifulSoup(html_content, 'lxml')
-        chunks = self._split_content(soup)
+        chunks = self._split_content(soup, self.chunk_size)
 
         markdown_chunks = []
         for chunk in chunks:

eveai_workers/Processors/pdf_processor.py

@@ -10,16 +10,17 @@ from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
 from common.utils.model_utils import create_language_template
 from .processor import Processor
+from common.utils.business_event_context import current_event
 
 
 class PDFProcessor(Processor):
     def __init__(self, tenant, model_variables, document_version):
         super().__init__(tenant, model_variables, document_version)
         # PDF-specific initialization
-        self.chunk_size = model_variables['PDF_chunk_size']
-        self.chunk_overlap = model_variables['PDF_chunk_overlap']
-        self.min_chunk_size = model_variables['PDF_min_chunk_size']
-        self.max_chunk_size = model_variables['PDF_max_chunk_size']
+        self.chunk_size = model_variables['processing_chunk_size']
+        self.chunk_overlap = model_variables['processing_chunk_overlap']
+        self.min_chunk_size = model_variables['processing_min_chunk_size']
+        self.max_chunk_size = model_variables['processing_max_chunk_size']
 
     def process(self):
         self._log("Starting PDF processing")
@@ -32,13 +33,14 @@ class PDFProcessor(Processor):
                 self.document_version.file_name
             )
 
-            extracted_content = self._extract_content(file_data)
-            structured_content, title = self._structure_content(extracted_content)
+            with current_event.create_span("PDF Extraction"):
+                extracted_content = self._extract_content(file_data)
+                structured_content, title = self._structure_content(extracted_content)
 
-            llm_chunks = self._split_content_for_llm(structured_content)
-            markdown = self._process_chunks_with_llm(llm_chunks)
-
-            self._save_markdown(markdown)
+            with current_event.create_span("Markdown Generation"):
+                llm_chunks = self._split_content_for_llm(structured_content)
+                markdown = self._process_chunks_with_llm(llm_chunks)
+                self._save_markdown(markdown)
             self._log("Finished processing PDF")
             return markdown, title
         except Exception as e:
@@ -228,12 +230,7 @@ class PDFProcessor(Processor):
         for chunk in chunks:
             input = {"pdf_content": chunk}
             result = chain.invoke(input)
-            # Remove Markdown code block delimiters if present
-            result = result.strip()
-            if result.startswith("```markdown"):
-                result = result[len("```markdown"):].strip()
-            if result.endswith("```"):
-                result = result[:-3].strip()
+            result = self._clean_markdown(result)
             markdown_chunks.append(result)
 
         return "\n\n".join(markdown_chunks)

eveai_workers/Processors/processor.py

@@ -40,3 +40,13 @@ class Processor(ABC):
             filename,
             content.encode('utf-8')
         )
+
+    def _clean_markdown(self, markdown):
+        markdown = markdown.strip()
+        if markdown.startswith("```markdown"):
+            markdown = markdown[len("```markdown"):].strip()
+        if markdown.endswith("```"):
+            markdown = markdown[:-3].strip()
+
+        return markdown
+

eveai_workers/Processors/srt_processor.py

@@ -1,37 +1,19 @@
-import re
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
-from common.utils.model_utils import create_language_template
-from .processor import Processor
+from .transcription_processor import TranscriptionProcessor
+import re
 
 
-class SRTProcessor(Processor):
-    def __init__(self, tenant, model_variables, document_version):
-        super().__init__(tenant, model_variables, document_version)
-
-    def process(self):
-        self._log("Starting SRT processing")
-        try:
-            file_data = minio_client.download_document_file(
-                self.tenant.id,
-                self.document_version.doc_id,
-                self.document_version.language,
-                self.document_version.id,
-                self.document_version.file_name
-            )
-
-            srt_content = file_data.decode('utf-8')
-            cleaned_transcription = self._clean_srt(srt_content)
-            markdown, title = self._generate_markdown_from_transcription(cleaned_transcription)
-
-            self._save_markdown(markdown)
-            self._log("Finished processing SRT")
-            return markdown, title
-        except Exception as e:
-            self._log(f"Error processing SRT: {str(e)}", level='error')
-            raise
+class SRTProcessor(TranscriptionProcessor):
+    def _get_transcription(self):
+        file_data = minio_client.download_document_file(
+            self.tenant.id,
+            self.document_version.doc_id,
+            self.document_version.language,
+            self.document_version.id,
+            self.document_version.file_name
+        )
+        srt_content = file_data.decode('utf-8')
+        return self._clean_srt(srt_content)
 
     def _clean_srt(self, srt_content):
         # Remove timecodes and subtitle numbers
@@ -50,31 +32,3 @@ class SRTProcessor(Processor):
 
         return cleaned_text
 
-    def _generate_markdown_from_transcription(self, transcription):
-        self._log("Generating markdown from transcription")
-        llm = self.model_variables['llm']
-        template = self.model_variables['transcript_template']
-        language_template = create_language_template(template, self.document_version.language)
-        transcript_prompt = ChatPromptTemplate.from_template(language_template)
-        setup = RunnablePassthrough()
-        output_parser = StrOutputParser()
-
-        chain = setup | transcript_prompt | llm | output_parser
-
-        input_transcript = {'transcript': transcription}
-        markdown = chain.invoke(input_transcript)
-
-        # Extract title from the markdown
-        title = self._extract_title_from_markdown(markdown)
-
-        return markdown, title
-
-    def _extract_title_from_markdown(self, markdown):
-        # Simple extraction of the first header as the title
-        lines = markdown.split('\n')
-        for line in lines:
-            if line.startswith('# '):
-                return line[2:].strip()
-        return "Untitled SRT Transcription"
-
-

eveai_workers/Processors/transcription_processor.py

@@ -0,0 +1,94 @@
+# transcription_processor.py
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+
+from common.utils.model_utils import create_language_template
+from .processor import Processor
+from common.utils.business_event_context import current_event
+
+
+class TranscriptionProcessor(Processor):
+    def __init__(self, tenant, model_variables, document_version):
+        super().__init__(tenant, model_variables, document_version)
+        self.chunk_size = model_variables['processing_chunk_size']
+        self.chunk_overlap = model_variables['processing_chunk_overlap']
+
+    def process(self):
+        self._log("Starting Transcription processing")
+        try:
+            with current_event.create_span("Transcription Generation"):
+                transcription = self._get_transcription()
+            with current_event.create_span("Markdown Generation"):
+                chunks = self._chunk_transcription(transcription)
+                markdown_chunks = self._process_chunks(chunks)
+                full_markdown = self._combine_markdown_chunks(markdown_chunks)
+                self._save_markdown(full_markdown)
+                self._log("Finished processing Transcription")
+            return full_markdown, self._extract_title_from_markdown(full_markdown)
+        except Exception as e:
+            self._log(f"Error processing Transcription: {str(e)}", level='error')
+            raise
+
+    def _get_transcription(self):
+        # This method should be implemented by child classes
+        raise NotImplementedError
+
+    def _chunk_transcription(self, transcription):
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function=len,
+            separators=["\n\n", "\n", " ", ""]
+        )
+        return text_splitter.split_text(transcription)
+
+    def _process_chunks(self, chunks):
+        self._log("Generating markdown from transcription")
+        llm = self.model_variables['llm']
+        template = self.model_variables['transcript_template']
+        language_template = create_language_template(template, self.document_version.language)
+        transcript_prompt = ChatPromptTemplate.from_template(language_template)
+        setup = RunnablePassthrough()
+        output_parser = StrOutputParser()
+
+        chain = setup | transcript_prompt | llm | output_parser
+
+        markdown_chunks = []
+        previous_part = ""
+        for i, chunk in enumerate(chunks):
+            self._log(f"Processing chunk {i + 1} of {len(chunks)}")
+            self._log(f"Previous part: {previous_part}")
+            input_transcript = {
+                'transcript': chunk,
+                'previous_part': previous_part
+            }
+            markdown = chain.invoke(input_transcript)
+            markdown = self._clean_markdown(markdown)
+            markdown_chunks.append(markdown)
+
+            # Extract the last part for the next iteration
+            lines = markdown.split('\n')
+            last_header = None
+            for line in reversed(lines):
+                if line.startswith('#'):
+                    last_header = line
+                    break
+            if last_header:
+                header_index = lines.index(last_header)
+                previous_part = '\n'.join(lines[header_index:])
+            else:
+                previous_part = lines[-1] if lines else ""
+
+        return markdown_chunks
+
+    def _combine_markdown_chunks(self, markdown_chunks):
+        return "\n\n".join(markdown_chunks)
+
+    def _extract_title_from_markdown(self, markdown):
+        lines = markdown.split('\n')
+        for line in lines:
+            if line.startswith('# '):
+                return line[2:].strip()
+        return "Untitled Transcription"

eveai_workers/tasks.py

@@ -24,79 +24,90 @@ from eveai_workers.Processors.html_processor import HTMLProcessor
 from eveai_workers.Processors.pdf_processor import PDFProcessor
 from eveai_workers.Processors.srt_processor import SRTProcessor
 
+from common.utils.business_event import BusinessEvent
+from common.utils.business_event_context import current_event
+
+
+# Healthcheck task
+@current_celery.task(name='ping', queue='embeddings')
+def ping():
+    return 'pong'
+
 
 @current_celery.task(name='create_embeddings', queue='embeddings')
 def create_embeddings(tenant_id, document_version_id):
-    current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id}.')
+    # BusinessEvent creates a context, which is why we need to use it with a with block
+    with BusinessEvent('Create Embeddings', tenant_id, document_version_id=document_version_id):
+        current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id}')
+        try:
+            # Retrieve Tenant for which we are processing
+            tenant = Tenant.query.get(tenant_id)
+            if tenant is None:
+                raise Exception(f'Tenant {tenant_id} not found')
 
-    try:
-        # Retrieve Tenant for which we are processing
-        tenant = Tenant.query.get(tenant_id)
-        if tenant is None:
-            raise Exception(f'Tenant {tenant_id} not found')
+            # Ensure we are working in the correct database schema
+            Database(tenant_id).switch_schema()
 
-        # Ensure we are working in the correct database schema
-        Database(tenant_id).switch_schema()
+            # Select variables to work with depending on tenant and model
+            model_variables = select_model_variables(tenant)
+            current_app.logger.debug(f'Model variables: {model_variables}')
 
-        # Select variables to work with depending on tenant and model
-        model_variables = select_model_variables(tenant)
-        current_app.logger.debug(f'Model variables: {model_variables}')
+            # Retrieve document version to process
+            document_version = DocumentVersion.query.get(document_version_id)
+            if document_version is None:
+                raise Exception(f'Document version {document_version_id} not found')
 
-        # Retrieve document version to process
-        document_version = DocumentVersion.query.get(document_version_id)
-        if document_version is None:
-            raise Exception(f'Document version {document_version_id} not found')
+        except Exception as e:
+            current_app.logger.error(f'Create Embeddings request received '
+                                     f'for non existing document version {document_version_id} '
+                                     f'for tenant {tenant_id}, '
+                                     f'error: {e}')
+            raise
 
-    except Exception as e:
-        current_app.logger.error(f'Create Embeddings request received '
-                                 f'for non existing document version {document_version_id} '
-                                 f'for tenant {tenant_id}, '
-                                 f'error: {e}')
-        raise
+        try:
+            db.session.add(document_version)
 
-    try:
-        db.session.add(document_version)
+            # start processing
+            document_version.processing = True
+            document_version.processing_started_at = dt.now(tz.utc)
+            document_version.processing_finished_at = None
+            document_version.processing_error = None
 
-        # start processing
-        document_version.processing = True
-        document_version.processing_started_at = dt.now(tz.utc)
-        document_version.processing_finished_at = None
-        document_version.processing_error = None
+            db.session.commit()
+        except SQLAlchemyError as e:
+            current_app.logger.error(f'Unable to save Embedding status information '
+                                     f'in document version {document_version_id} '
+                                     f'for tenant {tenant_id}')
+            raise
 
-        db.session.commit()
-    except SQLAlchemyError as e:
-        current_app.logger.error(f'Unable to save Embedding status information '
-                                 f'in document version {document_version_id} '
-                                 f'for tenant {tenant_id}')
-        raise
+        delete_embeddings_for_document_version(document_version)
 
-    delete_embeddings_for_document_version(document_version)
+        try:
+            match document_version.file_type:
+                case 'pdf':
+                    process_pdf(tenant, model_variables, document_version)
+                case 'html':
+                    process_html(tenant, model_variables, document_version)
+                case 'srt':
+                    process_srt(tenant, model_variables, document_version)
+                case 'mp4' | 'mp3' | 'ogg':
+                    process_audio(tenant, model_variables, document_version)
+                case _:
+                    raise Exception(f'No functionality defined for file type {document_version.file_type} '
+                                    f'for tenant {tenant_id} '
+                                    f'while creating embeddings for document version {document_version_id}')
+            current_event.log("Finished Embedding Creation Task")
 
-    try:
-        match document_version.file_type:
-            case 'pdf':
-                process_pdf(tenant, model_variables, document_version)
-            case 'html':
-                process_html(tenant, model_variables, document_version)
-            case 'srt':
-                process_srt(tenant, model_variables, document_version)
-            case 'mp4' | 'mp3' | 'ogg':
-                process_audio(tenant, model_variables, document_version)
-            case _:
-                raise Exception(f'No functionality defined for file type {document_version.file_type} '
-                                f'for tenant {tenant_id} '
-                                f'while creating embeddings for document version {document_version_id}')
-
-    except Exception as e:
-        current_app.logger.error(f'Error creating embeddings for tenant {tenant_id} '
-                                 f'on document version {document_version_id} '
-                                 f'error: {e}')
-        document_version.processing = False
-        document_version.processing_finished_at = dt.now(tz.utc)
-        document_version.processing_error = str(e)[:255]
-        db.session.commit()
-        create_embeddings.update_state(state=states.FAILURE)
-        raise
+        except Exception as e:
+            current_app.logger.error(f'Error creating embeddings for tenant {tenant_id} '
+                                     f'on document version {document_version_id} '
+                                     f'error: {e}')
+            document_version.processing = False
+            document_version.processing_finished_at = dt.now(tz.utc)
+            document_version.processing_error = str(e)[:255]
+            db.session.commit()
+            create_embeddings.update_state(state=states.FAILURE)
+            raise
 
 
 def delete_embeddings_for_document_version(document_version):
@@ -112,35 +123,43 @@ def delete_embeddings_for_document_version(document_version):
 
 
 def process_pdf(tenant, model_variables, document_version):
-    processor = PDFProcessor(tenant, model_variables, document_version)
-    markdown, title = processor.process()
+    with current_event.create_span("PDF Processing"):
+        processor = PDFProcessor(tenant, model_variables, document_version)
+        markdown, title = processor.process()
 
     # Process markdown and embed
-    embed_markdown(tenant, model_variables, document_version, markdown, title)
+    with current_event.create_span("Embedding"):
+        embed_markdown(tenant, model_variables, document_version, markdown, title)
 
 
 def process_html(tenant, model_variables, document_version):
-    processor = HTMLProcessor(tenant, model_variables, document_version)
-    markdown, title = processor.process()
+    with current_event.create_span("HTML Processing"):
+        processor = HTMLProcessor(tenant, model_variables, document_version)
+        markdown, title = processor.process()
 
     # Process markdown and embed
-    embed_markdown(tenant, model_variables, document_version, markdown, title)
+    with current_event.create_span("Embedding"):
+        embed_markdown(tenant, model_variables, document_version, markdown, title)
 
 
 def process_audio(tenant, model_variables, document_version):
-    processor = AudioProcessor(tenant, model_variables, document_version)
-    markdown, title = processor.process()
+    with current_event.create_span("Audio Processing"):
+        processor = AudioProcessor(tenant, model_variables, document_version)
+        markdown, title = processor.process()
 
     # Process markdown and embed
-    embed_markdown(tenant, model_variables, document_version, markdown, title)
+    with current_event.create_span("Embedding"):
+        embed_markdown(tenant, model_variables, document_version, markdown, title)
 
 
 def process_srt(tenant, model_variables, document_version):
-    processor = SRTProcessor(tenant, model_variables, document_version)
-    markdown, title = processor.process()
+    with current_event.create_span("SRT Processing"):
+        processor = SRTProcessor(tenant, model_variables, document_version)
+        markdown, title = processor.process()
 
     # Process markdown and embed
-    embed_markdown(tenant, model_variables, document_version, markdown, title)
+    with current_event.create_span("Embedding"):
+        embed_markdown(tenant, model_variables, document_version, markdown, title)
 
 
 def embed_markdown(tenant, model_variables, document_version, markdown, title):
@@ -175,6 +194,7 @@ def embed_markdown(tenant, model_variables, document_version, markdown, title):
 
 
 def enrich_chunks(tenant, model_variables, document_version, title, chunks):
+    current_event.log("Starting Enriching Chunks Processing")
     current_app.logger.debug(f'Enriching chunks for tenant {tenant.id} '
                              f'on document version {document_version.id}')
 
@@ -184,14 +204,21 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks):
 
     chunk_total_context = (f'Filename: {document_version.file_name}\n'
                            f'User Context:\n{document_version.user_context}\n\n'
+                           f'User Metadata:\n{document_version.user_metadata}\n\n'
                            f'Title: {title}\n'
-                           f'{summary}\n'
-                           f'{document_version.system_context}\n\n')
+                           f'Summary:\n{summary}\n'
+                           f'System Context:\n{document_version.system_context}\n\n'
+                           f'System Metadata:\n{document_version.system_metadata}\n\n'
+                           )
     enriched_chunks = []
     initial_chunk = (f'Filename: {document_version.file_name}\n'
                      f'User Context:\n{document_version.user_context}\n\n'
+                     f'User Metadata:\n{document_version.user_metadata}\n\n'
                      f'Title: {title}\n'
-                     f'{chunks[0]}')
+                     f'System Context:\n{document_version.system_context}\n\n'
+                     f'System Metadata:\n{document_version.system_metadata}\n\n'
+                     f'{chunks[0]}'
+                     )
 
     enriched_chunks.append(initial_chunk)
     for chunk in chunks[1:]:
@@ -200,11 +227,13 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks):
 
     current_app.logger.debug(f'Finished enriching chunks for tenant {tenant.id} '
                              f'on document version {document_version.id}')
+    current_event.log("Finished Enriching Chunks Processing")
 
     return enriched_chunks
 
 
 def summarize_chunk(tenant, model_variables, document_version, chunk):
+    current_event.log("Starting Summarizing Chunk")
     current_app.logger.debug(f'Summarizing chunk for tenant {tenant.id} '
                              f'on document version {document_version.id}')
     llm = model_variables['llm']
@@ -222,6 +251,7 @@ def summarize_chunk(tenant, model_variables, document_version, chunk):
         summary = chain.invoke({"text": chunk})
         current_app.logger.debug(f'Finished summarizing chunk for tenant {tenant.id} '
                                  f'on document version {document_version.id}.')
+        current_event.log("Finished Summarizing Chunk")
         return summary
     except LangChainException as e:
         current_app.logger.error(f'Error creating summary for chunk enrichment for tenant {tenant.id} '
@@ -231,6 +261,7 @@ def summarize_chunk(tenant, model_variables, document_version, chunk):
 
 
 def embed_chunks(tenant, model_variables, document_version, chunks):
+    current_event.log("Starting Embedding Chunks Processing")
     current_app.logger.debug(f'Embedding chunks for tenant {tenant.id} '
                              f'on document version {document_version.id}')
     embedding_model = model_variables['embedding_model']
@@ -255,6 +286,8 @@ def embed_chunks(tenant, model_variables, document_version, chunks):
         new_embedding.embedding = embedding
         new_embeddings.append(new_embedding)
 
+    current_app.logger.debug(f'Finished embedding chunks for tenant {tenant.id} ')
+
     return new_embeddings
 
 
@@ -268,244 +301,6 @@ def log_parsing_info(tenant, tags, included_elements, excluded_elements, exclude
         current_app.embed_tuning_logger.debug(f'First element to parse: {elements_to_parse[0]}')
 
 
-# def process_youtube(tenant, model_variables, document_version):
-#     download_file_name = f'{document_version.id}.mp4'
-#     compressed_file_name = f'{document_version.id}.mp3'
-#     transcription_file_name = f'{document_version.id}.txt'
-#     markdown_file_name = f'{document_version.id}.md'
-#
-#     # Remove existing files (in case of a re-processing of the file
-#     minio_client.delete_document_file(tenant.id, document_version.doc_id, document_version.language,
-#                                       document_version.id, download_file_name)
-#     minio_client.delete_document_file(tenant.id, document_version.doc_id, document_version.language,
-#                                       document_version.id, compressed_file_name)
-#     minio_client.delete_document_file(tenant.id, document_version.doc_id, document_version.language,
-#                                       document_version.id, transcription_file_name)
-#     minio_client.delete_document_file(tenant.id, document_version.doc_id, document_version.language,
-#                                       document_version.id, markdown_file_name)
-#
-#     of, title, description, author = download_youtube(document_version.url, tenant.id, document_version,
-#                                                       download_file_name)
-#     document_version.system_context = f'Title: {title}\nDescription: {description}\nAuthor: {author}'
-#     compress_audio(tenant.id, document_version, download_file_name, compressed_file_name)
-#     transcribe_audio(tenant.id, document_version, compressed_file_name, transcription_file_name, model_variables)
-#     annotate_transcription(tenant, document_version, transcription_file_name, markdown_file_name, model_variables)
-#
-#     potential_chunks = create_potential_chunks_for_markdown(tenant.id, document_version, markdown_file_name)
-#     actual_chunks = combine_chunks_for_markdown(potential_chunks, model_variables['min_chunk_size'],
-#                                                 model_variables['max_chunk_size'])
-#
-#     enriched_chunks = enrich_chunks(tenant, document_version, actual_chunks)
-#     embeddings = embed_chunks(tenant, model_variables, document_version, enriched_chunks)
-#
-#     try:
-#         db.session.add(document_version)
-#         document_version.processing_finished_at = dt.now(tz.utc)
-#         document_version.processing = False
-#         db.session.add_all(embeddings)
-#         db.session.commit()
-#     except SQLAlchemyError as e:
-#         current_app.logger.error(f'Error saving embedding information for tenant {tenant.id} '
-#                                  f'on Youtube document version {document_version.id}'
-#                                  f'error: {e}')
-#         raise
-#
-#     current_app.logger.info(f'Embeddings created successfully for tenant {tenant.id} '
-#                             f'on Youtube document version {document_version.id} :-)')
-#
-#
-# def download_youtube(url, tenant_id, document_version, file_name):
-#     try:
-#         current_app.logger.info(f'Downloading YouTube video: {url} for tenant: {tenant_id}')
-#         yt = YouTube(url)
-#         stream = yt.streams.get_audio_only()
-#
-#         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-#             stream.download(output_path=temp_file.name)
-#             with open(temp_file.name, 'rb') as f:
-#                 file_data = f.read()
-#
-#         minio_client.upload_document_file(tenant_id, document_version.doc_id, document_version.language,
-#                                           document_version.id,
-#                                           file_name, file_data)
-#
-#         current_app.logger.info(f'Downloaded YouTube video: {url} for tenant: {tenant_id}')
-#         return file_name, yt.title, yt.description, yt.author
-#     except Exception as e:
-#         current_app.logger.error(f'Error downloading YouTube video: {url} for tenant: {tenant_id} with error: {e}')
-#         raise
-#
-#
-# def compress_audio(tenant_id, document_version, input_file, output_file):
-#     try:
-#         current_app.logger.info(f'Compressing audio for tenant: {tenant_id}')
-#
-#         input_data = minio_client.download_document_file(tenant_id, document_version.doc_id, document_version.language,
-#                                                          document_version.id, input_file)
-#
-#         with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_input:
-#             temp_input.write(input_data)
-#             temp_input.flush()
-#
-#             with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as temp_output:
-#                 result = subprocess.run(
-#                     ['ffmpeg', '-i', temp_input.name, '-b:a', '64k', '-f', 'mp3', temp_output.name],
-#                     capture_output=True,
-#                     text=True
-#                 )
-#
-#                 if result.returncode != 0:
-#                     raise Exception(f"Compression failed: {result.stderr}")
-#
-#                 with open(temp_output.name, 'rb') as f:
-#                     compressed_data = f.read()
-#
-#         minio_client.upload_document_file(tenant_id, document_version.doc_id, document_version.language,
-#                                           document_version.id,
-#                                           output_file, compressed_data)
-#
-#         current_app.logger.info(f'Compressed audio for tenant: {tenant_id}')
-#     except Exception as e:
-#         current_app.logger.error(f'Error compressing audio for tenant: {tenant_id} with error: {e}')
-#         raise
-#
-#
-# def transcribe_audio(tenant_id, document_version, input_file, output_file, model_variables):
-#     try:
-#         current_app.logger.info(f'Transcribing audio for tenant: {tenant_id}')
-#         client = model_variables['transcription_client']
-#         model = model_variables['transcription_model']
-#
-#         # Download the audio file from MinIO
-#         audio_data = minio_client.download_document_file(tenant_id, document_version.doc_id, document_version.language,
-#                                                          document_version.id, input_file)
-#
-#         # Load the audio data into pydub
-#         audio = AudioSegment.from_mp3(io.BytesIO(audio_data))
-#
-#         # Define segment length (e.g., 10 minutes)
-#         segment_length = 10 * 60 * 1000  # 10 minutes in milliseconds
-#
-#         transcriptions = []
-#
-#         # Split audio into segments and transcribe each
-#         for i, chunk in enumerate(audio[::segment_length]):
-#             current_app.logger.debug(f'Transcribing chunk {i + 1} of {len(audio) // segment_length + 1}')
-#
-#             with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as temp_audio:
-#                 chunk.export(temp_audio.name, format="mp3")
-#
-#                 with open(temp_audio.name, 'rb') as audio_segment:
-#                     transcription = client.audio.transcriptions.create(
-#                         file=audio_segment,
-#                         model=model,
-#                         language=document_version.language,
-#                         response_format='verbose_json',
-#                     )
-#
-#                 transcriptions.append(transcription.text)
-#
-#             os.unlink(temp_audio.name)  # Delete the temporary file
-#
-#         # Combine all transcriptions
-#         full_transcription = " ".join(transcriptions)
-#
-#         # Upload the full transcription to MinIO
-#         minio_client.upload_document_file(
-#             tenant_id,
-#             document_version.doc_id,
-#             document_version.language,
-#             document_version.id,
-#             output_file,
-#             full_transcription.encode('utf-8')
-#         )
-#
-#         current_app.logger.info(f'Transcribed audio for tenant: {tenant_id}')
-#     except Exception as e:
-#         current_app.logger.error(f'Error transcribing audio for tenant: {tenant_id}, with error: {e}')
-#         raise
-#
-#
-# def annotate_transcription(tenant, document_version, input_file, output_file, model_variables):
-#     try:
-#         current_app.logger.debug(f'Annotating transcription for tenant {tenant.id}')
-#
-#         char_splitter = CharacterTextSplitter(separator='.',
-#                                               chunk_size=model_variables['annotation_chunk_length'],
-#                                               chunk_overlap=0)
-#
-#         headers_to_split_on = [
-#             ("#", "Header 1"),
-#             ("##", "Header 2"),
-#         ]
-#         markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on, strip_headers=False)
-#
-#         llm = model_variables['llm']
-#         template = model_variables['transcript_template']
-#         language_template = create_language_template(template, document_version.language)
-#         transcript_prompt = ChatPromptTemplate.from_template(language_template)
-#         setup = RunnablePassthrough()
-#         output_parser = StrOutputParser()
-#
-#         # Download the transcription file from MinIO
-#         transcript_data = minio_client.download_document_file(tenant.id, document_version.doc_id,
-#                                                               document_version.language, document_version.id,
-#                                                               input_file)
-#         transcript = transcript_data.decode('utf-8')
-#
-#         chain = setup | transcript_prompt | llm | output_parser
-#
-#         chunks = char_splitter.split_text(transcript)
-#         all_markdown_chunks = []
-#         last_markdown_chunk = ''
-#         for chunk in chunks:
-#             current_app.logger.debug(f'Annotating next chunk of {len(chunks)} for tenant {tenant.id}')
-#             full_input = last_markdown_chunk + '\n' + chunk
-#             if tenant.embed_tuning:
-#                 current_app.embed_tuning_logger.debug(f'Annotating chunk: \n '
-#                                                       f'------------------\n'
-#                                                       f'{full_input}\n'
-#                                                       f'------------------\n')
-#             input_transcript = {'transcript': full_input}
-#             markdown = chain.invoke(input_transcript)
-#             # GPT-4o returns some kind of content description: ```markdown <text> ```
-#             if markdown.startswith("```markdown"):
-#                 markdown = "\n".join(markdown.strip().split("\n")[1:-1])
-#             if tenant.embed_tuning:
-#                 current_app.embed_tuning_logger.debug(f'Markdown Received: \n '
-#                                                       f'------------------\n'
-#                                                       f'{markdown}\n'
-#                                                       f'------------------\n')
-#             md_header_splits = markdown_splitter.split_text(markdown)
-#             markdown_chunks = [doc.page_content for doc in md_header_splits]
-#             # claude-3.5-sonnet returns introductory text
-#             if not markdown_chunks[0].startswith('#'):
-#                 markdown_chunks.pop(0)
-#             last_markdown_chunk = markdown_chunks[-1]
-#             last_markdown_chunk = "\n".join(markdown.strip().split("\n")[1:])
-#             markdown_chunks.pop()
-#             all_markdown_chunks += markdown_chunks
-#
-#         all_markdown_chunks += [last_markdown_chunk]
-#
-#         annotated_transcript = '\n'.join(all_markdown_chunks)
-#
-#         # Upload the annotated transcript to MinIO
-#         minio_client.upload_document_file(
-#             tenant.id,
-#             document_version.doc_id,
-#             document_version.language,
-#             document_version.id,
-#             output_file,
-#             annotated_transcript.encode('utf-8')
-#         )
-#
-#         current_app.logger.info(f'Annotated transcription for tenant {tenant.id}')
-#     except Exception as e:
-#         current_app.logger.error(f'Error annotating transcription for tenant {tenant.id}, with error: {e}')
-#         raise
-
-
 def create_potential_chunks_for_markdown(tenant_id, document_version, input_file):
     try:
         current_app.logger.info(f'Creating potential chunks for tenant {tenant_id}')

migrations/public/versions/083ccd8206ea_add_tenant_type.py

@@ -0,0 +1,32 @@
+"""Add Tenant Type
+
+Revision ID: 083ccd8206ea
+Revises: ce6f5b62bbfb
+Create Date: 2024-09-12 11:30:41.958117
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '083ccd8206ea'
+down_revision = 'ce6f5b62bbfb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('type', sa.String(length=20), server_default='Active', nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.drop_column('type')
+
+    # ### end Alembic commands ###

migrations/public/versions/25588210dab2_llm_metrics_added.py

@@ -0,0 +1,49 @@
+"""LLM Metrics Added
+
+Revision ID: 25588210dab2
+Revises: 083ccd8206ea
+Create Date: 2024-09-17 12:44:12.242990
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '25588210dab2'
+down_revision = '083ccd8206ea'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('llm_usage_metric',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('tenant_id', sa.Integer(), nullable=False),
+    sa.Column('environment', sa.String(length=20), nullable=False),
+    sa.Column('activity', sa.String(length=20), nullable=False),
+    sa.Column('sub_activity', sa.String(length=20), nullable=False),
+    sa.Column('activity_detail', sa.String(length=50), nullable=True),
+    sa.Column('session_id', sa.String(length=50), nullable=True),
+    sa.Column('interaction_id', sa.Integer(), nullable=True),
+    sa.Column('document_version_id', sa.Integer(), nullable=True),
+    sa.Column('prompt_tokens', sa.Integer(), nullable=True),
+    sa.Column('completion_tokens', sa.Integer(), nullable=True),
+    sa.Column('total_tokens', sa.Integer(), nullable=True),
+    sa.Column('cost', sa.Float(), nullable=True),
+    sa.Column('latency', sa.Float(), nullable=True),
+    sa.Column('model_name', sa.String(length=50), nullable=False),
+    sa.Column('timestamp', sa.DateTime(), nullable=False),
+    sa.Column('additional_info', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    schema='public'
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('llm_usage_metric', schema='public')
+    # ### end Alembic commands ###

migrations/public/versions/2cbdb23ae02e_corrected_businesseventlog.py

@@ -0,0 +1,49 @@
+"""Corrected BusinessEventLog
+
+Revision ID: 2cbdb23ae02e
+Revises: e3c6ff8c22df
+Create Date: 2024-09-25 10:17:40.154566
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2cbdb23ae02e'
+down_revision = 'e3c6ff8c22df'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+        batch_op.alter_column('span_id',
+               existing_type=sa.VARCHAR(length=50),
+               nullable=True)
+        batch_op.alter_column('span_name',
+               existing_type=sa.VARCHAR(length=50),
+               nullable=True)
+        batch_op.alter_column('parent_span_id',
+               existing_type=sa.VARCHAR(length=50),
+               nullable=True)
+
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+        batch_op.alter_column('parent_span_id',
+               existing_type=sa.VARCHAR(length=50),
+               nullable=False)
+        batch_op.alter_column('span_name',
+               existing_type=sa.VARCHAR(length=50),
+               nullable=False)
+        batch_op.alter_column('span_id',
+               existing_type=sa.VARCHAR(length=50),
+               nullable=False)
+
+    # ### end Alembic commands ###

migrations/public/versions/829094f07d44_session_id_is_uuid_iso_integeger.py

@@ -0,0 +1,38 @@
+"""session_id is uuid iso integeger
+
+Revision ID: 829094f07d44
+Revises: 2cbdb23ae02e
+Create Date: 2024-09-27 09:19:13.201988
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '829094f07d44'
+down_revision = '2cbdb23ae02e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+        batch_op.alter_column('chat_session_id',
+               existing_type=sa.INTEGER(),
+               type_=sa.String(length=50),
+               existing_nullable=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('business_event_log', schema=None) as batch_op:
+        batch_op.alter_column('chat_session_id',
+               existing_type=sa.String(length=50),
+               type_=sa.INTEGER(),
+               existing_nullable=True)
+
+    # ### end Alembic commands ###

migrations/public/versions/e3c6ff8c22df_updated_monitoring_setup.py

@@ -0,0 +1,67 @@
+"""Updated Monitoring Setup
+
+Revision ID: e3c6ff8c22df
+Revises: 25588210dab2
+Create Date: 2024-09-25 10:05:57.684506
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e3c6ff8c22df'
+down_revision = '25588210dab2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('business_event_log',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('timestamp', sa.DateTime(), nullable=False),
+    sa.Column('event_type', sa.String(length=50), nullable=False),
+    sa.Column('tenant_id', sa.Integer(), nullable=False),
+    sa.Column('trace_id', sa.String(length=50), nullable=False),
+    sa.Column('span_id', sa.String(length=50), nullable=False),
+    sa.Column('span_name', sa.String(length=50), nullable=False),
+    sa.Column('parent_span_id', sa.String(length=50), nullable=False),
+    sa.Column('document_version_id', sa.Integer(), nullable=True),
+    sa.Column('chat_session_id', sa.Integer(), nullable=True),
+    sa.Column('interaction_id', sa.Integer(), nullable=True),
+    sa.Column('environment', sa.String(length=20), nullable=True),
+    sa.Column('message', sa.Text(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    schema='public'
+    )
+    op.drop_table('llm_usage_metric')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.create_table('llm_usage_metric',
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('tenant_id', sa.INTEGER(), autoincrement=False, nullable=False),
+    sa.Column('environment', sa.VARCHAR(length=20), autoincrement=False, nullable=False),
+    sa.Column('activity', sa.VARCHAR(length=20), autoincrement=False, nullable=False),
+    sa.Column('sub_activity', sa.VARCHAR(length=20), autoincrement=False, nullable=False),
+    sa.Column('activity_detail', sa.VARCHAR(length=50), autoincrement=False, nullable=True),
+    sa.Column('session_id', sa.VARCHAR(length=50), autoincrement=False, nullable=True),
+    sa.Column('interaction_id', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.Column('document_version_id', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.Column('prompt_tokens', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.Column('completion_tokens', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.Column('total_tokens', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.Column('cost', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True),
+    sa.Column('latency', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True),
+    sa.Column('model_name', sa.VARCHAR(length=50), autoincrement=False, nullable=False),
+    sa.Column('timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('additional_info', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name='llm_usage_metric_pkey')
+    )
+    op.drop_table('business_event_log', schema='public')
+    # ### end Alembic commands ###

nginx/nginx.conf

@@ -159,13 +159,12 @@ http {
         }
 
         location /flower/ {
-            proxy_pass http://127.0.0.1:5555/;
+            proxy_pass http://flower:5555/flower/;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
         }
-
     }
 
     include sites-enabled/*;

nginx/static/assets/css/eveai.css

@@ -508,3 +508,34 @@ input[type="radio"] {
     overflow-x: auto;
 }
 
+/* Hide Select2's custom elements */
+.select2-container-hidden {
+    position: absolute !important;
+    left: -9999px !important;
+}
+
+.select2-dropdown-hidden {
+    display: none !important;
+}
+
+/* Ensure the original select is visible and styled */
+select.select2 {
+    display: block !important;
+    width: 100% !important;
+    height: auto !important;
+    padding: .375rem .75rem !important;
+    font-size: 1rem !important;
+    line-height: 1.5 !important;
+    color: #495057 !important;
+    background-color: #fff !important;
+    background-clip: padding-box !important;
+    border: 1px solid #ced4da !important;
+    border-radius: .25rem !important;
+    transition: border-color .15s ease-in-out,box-shadow .15s ease-in-out !important;
+}
+
+/* Style for multiple select */
+select.select2[multiple] {
+    height: auto !important;
+}
+

project structure

@@ -1,70 +0,0 @@
-eveAI/
-│
-├── .venv/
-│
-├── common/
-│   ├── models/
-│   │   ├── __init__.py
-│   │   ├── document.py
-│   │   ├── interaction.py
-│   │   └── user.py
-│   │
-│   └── utils/
-│       ├── __init__.py
-│       └── extensions.py
-│
-├── config/
-│   ├── __init__.py
-│   ├── config.py
-│   └── logging_config.py
-│
-├── eveai_app/
-│   ├── static/
-│   ├── templates/
-│   │   ├── basic/
-│   │   ├── document/
-│   │   ├── interaction/
-│   │   ├── security/
-│   │   ├── user/
-│   │   ├── base.html
-│   │   ├── footer.html
-│   │   ├── head.html
-│   │   ├── header.html
-│   │   ├── index.html
-│   │   ├── macros.html
-│   │   ├── navbar.html
-│   │   ├── navbar_macros.html
-│   │   └── scripts.html
-│   │
-│   └── views/
-│       ├── __init__.py
-│       ├── basic_views.py
-│       ├── document_forms.py
-│       ├── document_views.py
-│       ├── errors.py
-│       ├── temp/
-│       ├── user_forms.py
-│       └── user_views.py
-│
-├── eveai_workers/
-│   ├── __init__.py
-│   ├── celery_utils.py
-│   └── tasks.py
-│
-├── instance/
-├── logs/
-│   ├── app.log
-│   ├── eveai.app.log
-│   └── eveai.workers.log
-│
-├── migrations/
-│
-├── scripts/
-│   ├── run_eveai_app.py
-│   ├── run_eveai_workers.py
-│   ├── start_eveai_app.sh
-│   ├── start_eveai_workers.sh
-│   ├── start_flower.sh
-│   └── start_logdy.sh
-│
-└── requirements.txt

repopack.config.json

@@ -0,0 +1,19 @@
+{
+  "output": {
+    "filePath": "eveai_repo.txt",
+    "style": "xml",
+    "removeComments": false,
+    "removeEmptyLines": false,
+    "topFilesLength": 5,
+    "showLineNumbers": false
+  },
+  "include": [],
+  "ignore": {
+    "useGitignore": true,
+    "useDefaultPatterns": true,
+    "customPatterns": []
+  },
+  "security": {
+    "enableSecurityCheck": true
+  }
+}

requirements.txt

@@ -26,22 +26,22 @@ greenlet~=3.0.3
 gunicorn~=22.0.0
 Jinja2~=3.1.4
 kombu~=5.3.7
-langchain~=0.2.7
-langchain-anthropic~=0.1.19
-langchain-community~=0.2.7
-langchain-core~=0.2.16
-langchain-mistralai~=0.1.9
-langchain-openai~=0.1.15
-langchain-postgres~=0.0.9
-langchain-text-splitters~=0.2.2
+langchain~=0.3.0
+langchain-anthropic~=0.2.0
+langchain-community~=0.3.0
+langchain-core~=0.3.0
+langchain-mistralai~=0.2.0
+langchain-openai~=0.2.0
+langchain-postgres~=0.0.12
+langchain-text-splitters~=0.3.0
 langcodes~=3.4.0
 langdetect~=1.0.9
 langsmith~=0.1.81
-openai~=1.35.13
+openai~=1.45.1
 pg8000~=1.31.2
 pgvector~=0.2.5
 pycryptodome~=3.20.0
-pydantic~=2.7.4
+pydantic~=2.9.1
 PyJWT~=2.8.0
 PySocks~=1.7.1
 python-dateutil~=2.9.0.post0
@@ -53,7 +53,7 @@ pytz~=2024.1
 PyYAML~=6.0.2rc1
 redis~=5.0.4
 requests~=2.32.3
-SQLAlchemy~=2.0.31
+SQLAlchemy~=2.0.35
 tiktoken~=0.7.0
 tzdata~=2024.1
 urllib3~=2.2.2
@@ -63,7 +63,7 @@ zxcvbn~=4.4.28
 groq~=0.9.0
 pydub~=0.25.1
 argparse~=1.4.0
-portkey_ai~=1.8.2
+portkey_ai~=1.8.7
 minio~=7.2.7
 Werkzeug~=3.0.3
 itsdangerous~=2.2.0
@@ -74,4 +74,9 @@ pillow~=10.4.0
 pdfplumber~=0.11.4
 PyPDF2~=3.0.1
 flask-restx~=1.3.0
-
+prometheus-flask-exporter~=0.23.1
+flask-healthz~=1.0.1
+langsmith~=0.1.121
+anthropic~=0.34.2
+prometheus-client~=0.20.0
+flower~=2.0.1

requirements.txt.freeze

@@ -1,175 +0,0 @@
-aiohttp==3.9.5
-aiosignal==1.3.1
-alembic==1.13.1
-amqp==5.2.0
-annotated-types==0.7.0
-anyio==4.4.0
-asn1crypto==1.5.1
-attrs==23.2.0
-Babel==2.15.0
-backoff==2.2.1
-bcrypt==4.1.3
-beautifulsoup4==4.12.3
-bidict==0.23.1
-billiard==4.2.0
-blinker==1.8.2
-cachelib==0.13.0
-cachetools==5.3.3
-celery==5.4.0
-certifi==2024.6.2
-chardet==5.2.0
-charset-normalizer==3.3.2
-click==8.1.7
-click-didyoumean==0.3.1
-click-plugins==1.1.1
-click-repl==0.3.0
-colorama==0.4.6
-cors==1.0.1
-dataclasses-json==0.6.7
-deepdiff==7.0.1
-distro==1.9.0
-dnspython==2.6.1
-dominate==2.9.1
-email_validator==2.2.0
-emoji==2.12.1
-eventlet==0.36.1
-filelock==3.15.4
-filetype==1.2.0
-Flask==3.0.3
-Flask-BabelEx==0.9.4
-Flask-Bootstrap==3.3.7.1
-Flask-Cors==4.0.1
-Flask-JWT-Extended==4.6.0
-Flask-Login==0.6.3
-flask-mailman==1.1.0
-Flask-Migrate==4.0.7
-Flask-Principal==0.4.0
-Flask-Security-Too==5.4.3
-Flask-Session==0.8.0
-Flask-SocketIO==5.3.6
-Flask-SQLAlchemy==3.1.1
-Flask-WTF==1.2.1
-flower==2.0.1
-frozenlist==1.4.1
-fsspec==2024.6.0
-future==1.0.0
-gevent==24.2.1
-gevent-websocket==0.10.1
-google==3.0.0
-google-api-core==2.19.1rc0
-google-auth==2.30.0
-google-cloud-core==2.4.1
-google-cloud-kms==2.23.0
-googleapis-common-protos==1.63.2rc0
-greenlet==3.0.3
-grpc-google-iam-v1==0.13.0
-grpcio==1.63.0
-grpcio-status==1.62.2
-gunicorn==22.0.0
-h11==0.14.0
-httpcore==1.0.5
-httpx==0.27.0
-httpx-sse==0.4.0
-huggingface-hub==0.23.4
-humanize==4.9.0
-idna==3.7
-importlib_resources==6.4.0
-itsdangerous==2.2.0
-Jinja2==3.1.4
-joblib==1.4.2
-jsonpatch==1.33
-jsonpath-python==1.0.6
-jsonpointer==3.0.0
-kombu==5.3.7
-langchain==0.2.5
-langchain-community==0.2.5
-langchain-core==0.2.9
-langchain-mistralai==0.1.8
-langchain-openai==0.1.9
-langchain-postgres==0.0.9
-langchain-text-splitters==0.2.1
-langcodes==3.4.0
-langdetect==1.0.9
-langsmith==0.1.81
-language_data==1.2.0
-lxml==5.2.2
-Mako==1.3.5
-marisa-trie==1.2.0
-MarkupSafe==2.1.5
-marshmallow==3.21.3
-msgspec==0.18.6
-multidict==6.0.5
-mypy-extensions==1.0.0
-nest-asyncio==1.6.0
-nltk==3.8.1
-numpy==2.0.0
-openai==1.35.3
-ordered-set==4.1.0
-orjson==3.10.5
-packaging==24.1
-passlib==1.7.4
-pg8000==1.31.2
-pgvector==0.2.5
-prometheus_client==0.20.0
-prompt_toolkit==3.0.47
-proto-plus==1.24.0
-protobuf==5.27.1
-psycopg==3.1.19
-psycopg-pool==3.2.2
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
-pycryptodome==3.20.0
-pydantic==2.7.4
-pydantic_core==2.19.0
-pydevd-pycharm==242.18071.12
-PyJWT==2.8.0
-pypdf==4.2.0
-PySocks==1.7.1
-python-dateutil==2.9.0.post0
-python-engineio==4.9.1
-python-iso639==2024.4.27
-python-magic==0.4.27
-python-socketio==5.11.3
-pytz==2024.1
-PyYAML==6.0.2rc1
-rapidfuzz==3.9.3
-redis==5.0.4
-regex==2024.4.28
-requests==2.32.3
-requests-file==2.1.0
-requests-toolbelt==1.0.0
-rsa==4.9
-scramp==1.4.5
-setuptools==69.5.1
-simple-websocket==1.0.0
-six==1.16.0
-sniffio==1.3.1
-soupsieve==2.5
-speaklater==1.3
-SQLAlchemy==2.0.31
-tabulate==0.9.0
-tenacity==8.4.2
-tiktoken==0.7.0
-tldextract==5.1.2
-tokenizers==0.19.1
-tornado==6.4.1
-tqdm==4.66.4
-typing-inspect==0.9.0
-typing_extensions==4.12.2
-tzdata==2024.1
-unstructured==0.14.8
-unstructured-client==0.23.7
-urllib3==2.2.2
-uWSGI==2.0.26
-vine==5.1.0
-visitor==0.1.3
-wcwidth==0.2.13
-Werkzeug==3.0.3
-wrapt==1.16.0
-wsproto==1.2.0
-WTForms==3.1.2
-wtforms-html5==0.6.1
-yarl==1.9.4
-zope.event==5.0
-zope.interface==6.3
-zxcvbn==4.4.28

requirements_orig.txt

@@ -1,19 +0,0 @@
-Flask~=3.0.3
-WTForms~=3.1.2
-SQLAlchemy~=2.0.29
-alembic~=1.13.1
-Werkzeug~=3.0.2
-pgvector~=0.2.5
-gevent~=24.2.1
-celery~=5.4.0
-kombu~=5.3.7
-langchain~=0.1.17
-requests~=2.31.0
-beautifulsoup4~=4.12.3
-google~=3.0.0
-redis~=5.0.4
-itsdangerous~=2.2.0
-pydantic~=2.7.1
-chardet~=5.2.0
-langcodes~=3.4.0
-pytz~=2024.1

scripts/start_eveai_workers.sh

@@ -8,7 +8,7 @@ export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR"  # In
 chown -R appuser:appuser /app/logs
 
 # Start a worker for the 'embeddings' queue with higher concurrency
-celery -A eveai_workers.celery worker --loglevel=info -Q embeddings --autoscale=2,8 --hostname=embeddings_worker@%h &
+celery -A eveai_workers.celery worker --loglevel=debug -Q embeddings --autoscale=2,8 --hostname=embeddings_worker@%h &
 
 # Start a worker for the 'llm_interactions' queue with auto-scaling - not necessary, in eveai_chat_workers
 # celery -A eveai_workers.celery worker --loglevel=info - Q llm_interactions --autoscale=2,8 --hostname=interactions_worker@%h &

scripts/start_flower.sh

@@ -1,9 +1,28 @@
-#!/usr/bin/env bash
+#!/bin/bash
+set -e
 
-cd "/Volumes/OWC4M2_1/Dropbox/Josako's Dev/Josako/EveAI/Development/eveAI/" || exit 1
-source "/Volumes/OWC4M2_1/Dropbox/Josako's Dev/Josako/EveAI/Development/eveAI/.venv/bin/activate"
+# scripts/start_flower.sh
 
-# on development machine, no authentication required
-export FLOWER_UNAUTHENTICATED_API=True
-# Start a worker for the 'embeddings' queue with higher concurrency
-celery -A eveai_workers.celery flower
+# Set default values
+REDIS_HOST=${REDIS_URL:-redis}
+REDIS_PORT=${REDIS_PORT:-6379}
+
+# Set environment-specific variables
+if [ "$FLASK_ENV" = "production" ]; then
+    # Production settings
+    export FLOWER_BASIC_AUTH="${FLOWER_USER}:${FLOWER_PASSWORD}"
+    export FLOWER_BROKER_URL="redis://${REDIS_USER}:${REDIS_PASS}@${REDIS_URL}:${REDIS_PORT}/0"
+    export CELERY_BROKER_URL="redis://${REDIS_USER}:${REDIS_PASS}@${REDIS_URL}:${REDIS_PORT}/0"
+else
+    # Development settings
+    export FLOWER_BROKER_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
+    export CELERY_BROKER_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
+fi
+
+echo $BROKER_URL
+echo "----------"
+
+# Start Flower
+exec celery flower \
+    --url-prefix=/flower \
+    --port=5555