From 18074353399b647963ea64ef459e86c5450bc1f8 Mon Sep 17 00:00:00 2001 From: Josako Date: Fri, 15 Nov 2024 10:00:53 +0100 Subject: [PATCH] - Introduction of dynamic Retrievers & Specialists - Introduction of dynamic Processors - Introduction of caching system - Introduction of a better template manager - Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists - Start adaptation of chat client --- .gitignore | 2 + common/extensions.py | 4 + common/langchain/outputs/base.py | 23 + common/langchain/outputs/rag.py | 22 + .../retrievers/eveai_default_rag_retriever.py | 145 ----- .../retrievers/eveai_dossier_retriever.py | 154 ----- .../retrievers/eveai_history_retriever.py | 52 -- .../langchain/retrievers/eveai_retriever.py | 40 -- .../langchain/templates/template_manager.py | 154 +++++ common/langchain/tracked_transcribe.py | 27 - common/langchain/tracked_transcription.py | 77 +++ common/models/document.py | 32 +- common/models/interaction.py | 55 +- common/utils/business_event.py | 1 - .../cache/__init__old.py} | 0 common/utils/cache/base.py | 89 +++ common/utils/cache/eveai_cache_manager.py | 32 + common/utils/cache/regions.py | 61 ++ common/utils/celery_utils.py | 2 - common/utils/config_field_types.py | 613 ++++++++++++++++++ common/utils/cors_utils.py | 15 - common/utils/debug_utils.py | 3 +- common/utils/document_utils.py | 8 +- common/utils/eveai_exceptions.py | 7 + common/utils/middleware.py | 3 - common/utils/model_utils.py | 503 +++++++------- common/utils/os_utils.py | 16 + common/utils/security.py | 1 - common/utils/security_utils.py | 7 +- common/utils/string_list_converter.py | 112 ++++ common/utils/view_assistants.py | 2 +- config/catalog_types.py | 28 +- config/config.py | 28 +- config/logging_config.py | 234 ++++++- config/processor_types.py | 56 ++ .../prompts/anthropic/claude-3-5-sonnet.yaml | 88 --- config/prompts/openai/gpt-4o-mini.yaml | 79 --- config/prompts/openai/gpt-4o.yaml | 84 --- .../openai/gpt-4o/encyclopedia/1.0.0.yaml | 12 + .../prompts/openai/gpt-4o/history/1.0.0.yaml | 16 + .../openai/gpt-4o/html_parse/1.0.0.yaml | 20 + .../openai/gpt-4o/pdf_parse/1.0.0.yaml | 23 + config/prompts/openai/gpt-4o/rag/1.0.0.yaml | 15 + .../prompts/openai/gpt-4o/summary/1.0.0.yaml | 9 + .../openai/gpt-4o/transcript/1.0.0.yaml | 25 + config/retriever_types.py | 8 + config/specialist_types.py | 41 +- docker/compose_dev.yaml | 19 +- docker/compose_stackhero.yaml | 1 - eveai_api/__init__.py | 23 +- eveai_api/api/auth.py | 8 - eveai_app/__init__.py | 3 +- .../templates/document/edit_processor.html | 33 + eveai_app/templates/document/processor.html | 23 + eveai_app/templates/document/processors.html | 23 + .../interaction/edit_specialist.html | 2 +- eveai_app/templates/navbar.html | 2 + eveai_app/views/basic_views.py | 3 +- eveai_app/views/document_forms.py | 117 ++-- eveai_app/views/document_views.py | 172 +++-- eveai_app/views/dynamic_form_base.py | 29 +- eveai_app/views/entitlements_views.py | 1 - eveai_app/views/interaction_forms.py | 2 +- eveai_app/views/interaction_views.py | 40 +- eveai_app/views/security_views.py | 31 +- eveai_app/views/user_views.py | 20 +- eveai_chat/socket_handlers/chat_handler.py | 37 +- eveai_chat_workers/__init__.py | 12 +- eveai_chat_workers/chat_session_cache.py | 193 ++++++ eveai_chat_workers/retrievers/__init__.py | 5 + eveai_chat_workers/retrievers/base.py | 57 ++ eveai_chat_workers/retrievers/registry.py | 20 + .../retrievers/retriever_typing.py | 60 ++ eveai_chat_workers/retrievers/standard_rag.py | 140 ++++ eveai_chat_workers/specialists/__init__.py | 5 + eveai_chat_workers/specialists/base.py | 50 ++ .../specialists/rag_specialist.py | 289 +++++++++ eveai_chat_workers/specialists/registry.py | 21 + .../specialists/specialist_typing.py | 144 ++++ eveai_chat_workers/tasks.py | 512 ++++++++------- eveai_entitlements/tasks.py | 9 - eveai_workers/__init__.py | 11 +- eveai_workers/processors/__init__.py | 5 + .../audio_processor.py | 75 ++- .../base_processor.py} | 42 +- .../html_processor.py | 51 +- .../pdf_processor.py | 43 +- .../processors/processor_registry.py | 92 +++ .../srt_processor.py | 4 +- .../transcription_processor.py | 30 +- eveai_workers/tasks.py | 198 +++--- .../eveai-chat-widget/eveai-chat_plugin.php | 7 +- .../eveai-chat-widget/js/eveai-chat-widget.js | 37 +- .../eveai-chat-widget/js/eveai-sdk.js | 6 +- ...llow_json_specialist_input_and_results_.py | 31 + ...0e5_add_sub_file_type_to_processor_and_.py | 31 + ...6c_removed_obsolete_interaction_fields_.py | 41 ++ .../95f82a301dff_adding_processor_model.py | 49 ++ ...30a88c9_remove_obsolete_columns_due_to_.py | 47 ++ nginx/public/chat_evie.html | 3 +- requirements.txt | 8 +- 101 files changed, 4181 insertions(+), 1764 deletions(-) create mode 100644 common/langchain/outputs/base.py create mode 100644 common/langchain/outputs/rag.py delete mode 100644 common/langchain/retrievers/eveai_default_rag_retriever.py delete mode 100644 common/langchain/retrievers/eveai_dossier_retriever.py delete mode 100644 common/langchain/retrievers/eveai_history_retriever.py delete mode 100644 common/langchain/retrievers/eveai_retriever.py create mode 100644 common/langchain/templates/template_manager.py delete mode 100644 common/langchain/tracked_transcribe.py create mode 100644 common/langchain/tracked_transcription.py rename common/{langchain/retrievers/__init__.py => utils/cache/__init__old.py} (100%) create mode 100644 common/utils/cache/base.py create mode 100644 common/utils/cache/eveai_cache_manager.py create mode 100644 common/utils/cache/regions.py create mode 100644 common/utils/config_field_types.py create mode 100644 common/utils/string_list_converter.py create mode 100644 config/processor_types.py delete mode 100644 config/prompts/anthropic/claude-3-5-sonnet.yaml delete mode 100644 config/prompts/openai/gpt-4o-mini.yaml delete mode 100644 config/prompts/openai/gpt-4o.yaml create mode 100644 config/prompts/openai/gpt-4o/encyclopedia/1.0.0.yaml create mode 100644 config/prompts/openai/gpt-4o/history/1.0.0.yaml create mode 100644 config/prompts/openai/gpt-4o/html_parse/1.0.0.yaml create mode 100644 config/prompts/openai/gpt-4o/pdf_parse/1.0.0.yaml create mode 100644 config/prompts/openai/gpt-4o/rag/1.0.0.yaml create mode 100644 config/prompts/openai/gpt-4o/summary/1.0.0.yaml create mode 100644 config/prompts/openai/gpt-4o/transcript/1.0.0.yaml create mode 100644 eveai_app/templates/document/edit_processor.html create mode 100644 eveai_app/templates/document/processor.html create mode 100644 eveai_app/templates/document/processors.html create mode 100644 eveai_chat_workers/chat_session_cache.py create mode 100644 eveai_chat_workers/retrievers/__init__.py create mode 100644 eveai_chat_workers/retrievers/base.py create mode 100644 eveai_chat_workers/retrievers/registry.py create mode 100644 eveai_chat_workers/retrievers/retriever_typing.py create mode 100644 eveai_chat_workers/retrievers/standard_rag.py create mode 100644 eveai_chat_workers/specialists/__init__.py create mode 100644 eveai_chat_workers/specialists/base.py create mode 100644 eveai_chat_workers/specialists/rag_specialist.py create mode 100644 eveai_chat_workers/specialists/registry.py create mode 100644 eveai_chat_workers/specialists/specialist_typing.py create mode 100644 eveai_workers/processors/__init__.py rename eveai_workers/{Processors => processors}/audio_processor.py (74%) rename eveai_workers/{Processors/processor.py => processors/base_processor.py} (50%) rename eveai_workers/{Processors => processors}/html_processor.py (76%) rename eveai_workers/{Processors => processors}/pdf_processor.py (87%) create mode 100644 eveai_workers/processors/processor_registry.py rename eveai_workers/{Processors => processors}/srt_processor.py (90%) rename eveai_workers/{Processors => processors}/transcription_processor.py (80%) create mode 100644 migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py create mode 100644 migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py create mode 100644 migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py create mode 100644 migrations/tenant/versions/95f82a301dff_adding_processor_model.py create mode 100644 migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py diff --git a/.gitignore b/.gitignore index 7e7f193..1d6f441 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ scripts/.DS_Store scripts/__pycache__/run_eveai_app.cpython-312.pyc /eveai_repo.txt *repo.txt +/docker/eveai_logs/ +/common/utils/model_utils_orig.py diff --git a/common/extensions.py b/common/extensions.py index 91d28e5..e3e2f78 100644 --- a/common/extensions.py +++ b/common/extensions.py @@ -12,6 +12,8 @@ from flask_wtf import CSRFProtect from flask_restx import Api from prometheus_flask_exporter import PrometheusMetrics +from .langchain.templates.template_manager import TemplateManager +from .utils.cache.eveai_cache_manager import EveAICacheManager from .utils.simple_encryption import SimpleEncryption from .utils.minio_utils import MinioClient @@ -32,3 +34,5 @@ api_rest = Api() simple_encryption = SimpleEncryption() minio_client = MinioClient() metrics = PrometheusMetrics.for_app_factory() +template_manager = TemplateManager() +cache_manager = EveAICacheManager() diff --git a/common/langchain/outputs/base.py b/common/langchain/outputs/base.py new file mode 100644 index 0000000..0af84d4 --- /dev/null +++ b/common/langchain/outputs/base.py @@ -0,0 +1,23 @@ +# Output Schema Management - common/langchain/outputs/base.py +from typing import Dict, Type, Any +from pydantic import BaseModel + + +class BaseSpecialistOutput(BaseModel): + """Base class for all specialist outputs""" + pass + + +class OutputRegistry: + """Registry for specialist output schemas""" + _schemas: Dict[str, Type[BaseSpecialistOutput]] = {} + + @classmethod + def register(cls, specialist_type: str, schema_class: Type[BaseSpecialistOutput]): + cls._schemas[specialist_type] = schema_class + + @classmethod + def get_schema(cls, specialist_type: str) -> Type[BaseSpecialistOutput]: + if specialist_type not in cls._schemas: + raise ValueError(f"No output schema registered for {specialist_type}") + return cls._schemas[specialist_type] diff --git a/common/langchain/outputs/rag.py b/common/langchain/outputs/rag.py new file mode 100644 index 0000000..1f86429 --- /dev/null +++ b/common/langchain/outputs/rag.py @@ -0,0 +1,22 @@ +# RAG Specialist Output - common/langchain/outputs/rag.py +from typing import List +from pydantic import Field +from .base import BaseSpecialistOutput + + +class RAGOutput(BaseSpecialistOutput): + """Output schema for RAG specialist""" + """Default docstring - to be replaced with actual prompt""" + + answer: str = Field( + ..., + description="The answer to the user question, based on the given sources", + ) + citations: List[int] = Field( + ..., + description="The integer IDs of the SPECIFIC sources that were used to generate the answer" + ) + insufficient_info: bool = Field( + False, # Default value is set to False + description="A boolean indicating whether given sources were sufficient or not to generate the answer" + ) diff --git a/common/langchain/retrievers/eveai_default_rag_retriever.py b/common/langchain/retrievers/eveai_default_rag_retriever.py deleted file mode 100644 index 8d03587..0000000 --- a/common/langchain/retrievers/eveai_default_rag_retriever.py +++ /dev/null @@ -1,145 +0,0 @@ -from langchain_core.retrievers import BaseRetriever -from sqlalchemy import func, and_, or_, desc -from sqlalchemy.exc import SQLAlchemyError -from pydantic import BaseModel, Field, PrivateAttr -from typing import Any, Dict -from flask import current_app - -from common.extensions import db -from common.models.document import Document, DocumentVersion -from common.utils.datetime_utils import get_date_in_timezone -from common.utils.model_utils import ModelVariables - - -class EveAIDefaultRagRetriever(BaseRetriever, BaseModel): - _catalog_id: int = PrivateAttr() - _model_variables: ModelVariables = PrivateAttr() - _tenant_info: Dict[str, Any] = PrivateAttr() - - def __init__(self, catalog_id: int, model_variables: ModelVariables, tenant_info: Dict[str, Any]): - super().__init__() - current_app.logger.debug(f'Model variables type: {type(model_variables)}') - self._catalog_id = catalog_id - self._model_variables = model_variables - self._tenant_info = tenant_info - - @property - def catalog_id(self) -> int: - return self._catalog_id - - @property - def model_variables(self) -> ModelVariables: - return self._model_variables - - @property - def tenant_info(self) -> Dict[str, Any]: - return self._tenant_info - - def _get_relevant_documents(self, query: str): - current_app.logger.debug(f'Retrieving relevant documents for query: {query}') - query_embedding = self._get_query_embedding(query) - current_app.logger.debug(f'Model Variables Private: {type(self._model_variables)}') - current_app.logger.debug(f'Model Variables Property: {type(self.model_variables)}') - db_class = self.model_variables['embedding_db_model'] - similarity_threshold = self.model_variables['similarity_threshold'] - k = self.model_variables['k'] - - if self.model_variables['rag_tuning']: - try: - current_date = get_date_in_timezone(self.tenant_info['timezone']) - current_app.rag_tuning_logger.debug(f'Current date: {current_date}\n') - - # Debug query to show similarity for all valid documents (without chunk text) - debug_query = ( - db.session.query( - Document.id.label('document_id'), - DocumentVersion.id.label('version_id'), - db_class.id.label('embedding_id'), - (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity') - ) - .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id) - .join(Document, DocumentVersion.doc_id == Document.id) - .filter( - or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date), - or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date) - ) - .order_by(desc('similarity')) - ) - - debug_results = debug_query.all() - - current_app.logger.debug("Debug: Similarity for all valid documents:") - for row in debug_results: - current_app.rag_tuning_logger.debug(f"Doc ID: {row.document_id}, " - f"Version ID: {row.version_id}, " - f"Embedding ID: {row.embedding_id}, " - f"Similarity: {row.similarity}") - current_app.rag_tuning_logger.debug(f'---------------------------------------\n') - except SQLAlchemyError as e: - current_app.logger.error(f'Error generating overview: {e}') - db.session.rollback() - - if self.model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Parameters for Retrieval of documents: \n') - current_app.rag_tuning_logger.debug(f'Similarity Threshold: {similarity_threshold}\n') - current_app.rag_tuning_logger.debug(f'K: {k}\n') - current_app.rag_tuning_logger.debug(f'---------------------------------------\n') - - try: - current_date = get_date_in_timezone(self.tenant_info['timezone']) - # Subquery to find the latest version of each document - subquery = ( - db.session.query( - DocumentVersion.doc_id, - func.max(DocumentVersion.id).label('latest_version_id') - ) - .group_by(DocumentVersion.doc_id) - .subquery() - ) - # Main query to filter embeddings - query_obj = ( - db.session.query(db_class, - (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity')) - .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id) - .join(Document, DocumentVersion.doc_id == Document.id) - .join(subquery, DocumentVersion.id == subquery.c.latest_version_id) - .filter( - or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date), - or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date), - (1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold, - Document.catalog_id == self._catalog_id - ) - .order_by(desc('similarity')) - .limit(k) - ) - - if self.model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Query executed for Retrieval of documents: \n') - current_app.rag_tuning_logger.debug(f'{query_obj.statement}\n') - current_app.rag_tuning_logger.debug(f'---------------------------------------\n') - - res = query_obj.all() - - if self.model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Retrieved {len(res)} relevant documents \n') - current_app.rag_tuning_logger.debug(f'Data retrieved: \n') - current_app.rag_tuning_logger.debug(f'{res}\n') - current_app.rag_tuning_logger.debug(f'---------------------------------------\n') - - result = [] - for doc in res: - if self.model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n') - current_app.rag_tuning_logger.debug(f'Chunk: \n {doc[0].chunk}\n\n') - result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n') - - except SQLAlchemyError as e: - current_app.logger.error(f'Error retrieving relevant documents: {e}') - db.session.rollback() - return [] - return result - - def _get_query_embedding(self, query: str): - embedding_model = self.model_variables['embedding_model'] - query_embedding = embedding_model.embed_query(query) - return query_embedding diff --git a/common/langchain/retrievers/eveai_dossier_retriever.py b/common/langchain/retrievers/eveai_dossier_retriever.py deleted file mode 100644 index e54b8a6..0000000 --- a/common/langchain/retrievers/eveai_dossier_retriever.py +++ /dev/null @@ -1,154 +0,0 @@ -from langchain_core.retrievers import BaseRetriever -from sqlalchemy import func, and_, or_, desc, cast, JSON -from sqlalchemy.exc import SQLAlchemyError -from pydantic import BaseModel, Field, PrivateAttr -from typing import Any, Dict, List, Optional -from flask import current_app -from contextlib import contextmanager - -from common.extensions import db -from common.models.document import Document, DocumentVersion, Catalog -from common.utils.datetime_utils import get_date_in_timezone -from common.utils.model_utils import ModelVariables - - -class EveAIDossierRetriever(BaseRetriever, BaseModel): - _catalog_id: int = PrivateAttr() - _model_variables: ModelVariables = PrivateAttr() - _tenant_info: Dict[str, Any] = PrivateAttr() - _active_filters: Optional[Dict[str, Any]] = PrivateAttr() - - def __init__(self, catalog_id: int, model_variables: ModelVariables, tenant_info: Dict[str, Any]): - super().__init__() - self._catalog_id = catalog_id - self._model_variables = model_variables - self._tenant_info = tenant_info - self._active_filters = None - - @contextmanager - def filtering(self, metadata_filters: Dict[str, Any]): - """Context manager for temporarily setting metadata filters""" - previous_filters = self._active_filters - self._active_filters = metadata_filters - try: - yield self - finally: - self._active_filters = previous_filters - - def _build_metadata_filter_conditions(self, query): - """Build SQL conditions for metadata filtering""" - if not self._active_filters: - return query - - conditions = [] - for field, value in self._active_filters.items(): - if value is None: - continue - - # Handle both single values and lists of values - if isinstance(value, (list, tuple)): - # Multiple values - create OR condition - or_conditions = [] - for val in value: - or_conditions.append( - cast(DocumentVersion.user_metadata[field].astext, JSON) == str(val) - ) - if or_conditions: - conditions.append(or_(*or_conditions)) - else: - # Single value - direct comparison - conditions.append( - cast(DocumentVersion.user_metadata[field].astext, JSON) == str(value) - ) - - if conditions: - query = query.filter(and_(*conditions)) - - return query - - def _get_relevant_documents(self, query: str): - current_app.logger.debug(f'Retrieving relevant documents for dossier query: {query}') - if self._active_filters: - current_app.logger.debug(f'Using metadata filters: {self._active_filters}') - - query_embedding = self._get_query_embedding(query) - db_class = self.model_variables['embedding_db_model'] - similarity_threshold = self.model_variables['similarity_threshold'] - k = self.model_variables['k'] - - try: - current_date = get_date_in_timezone(self.tenant_info['timezone']) - - # Subquery to find the latest version of each document - subquery = ( - db.session.query( - DocumentVersion.doc_id, - func.max(DocumentVersion.id).label('latest_version_id') - ) - .group_by(DocumentVersion.doc_id) - .subquery() - ) - - # Build base query - # Build base query - query_obj = ( - db.session.query(db_class, - (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity')) - .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id) - .join(Document, DocumentVersion.doc_id == Document.id) - .join(subquery, DocumentVersion.id == subquery.c.latest_version_id) - .filter( - or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date), - or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date), - (1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold, - Document.catalog_id == self._catalog_id - ) - ) - - # Apply metadata filters - query_obj = self._build_metadata_filter_conditions(query_obj) - - # Order and limit results - query_obj = query_obj.order_by(desc('similarity')).limit(k) - - # Debug logging for RAG tuning if enabled - if self.model_variables['rag_tuning']: - self._log_rag_tuning(query_obj, query_embedding) - - res = query_obj.all() - - result = [] - for doc in res: - if self.model_variables['rag_tuning']: - current_app.logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n') - current_app.logger.debug(f'Chunk: \n {doc[0].chunk}\n\n') - result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n') - - except SQLAlchemyError as e: - current_app.logger.error(f'Error retrieving relevant documents: {e}') - db.session.rollback() - return [] - - return result - - def _log_rag_tuning(self, query_obj, query_embedding): - """Log debug information for RAG tuning""" - current_app.rag_tuning_logger.debug("Debug: Query execution plan:") - current_app.rag_tuning_logger.debug(f"{query_obj.statement}") - if self._active_filters: - current_app.rag_tuning_logger.debug("Debug: Active metadata filters:") - current_app.rag_tuning_logger.debug(f"{self._active_filters}") - - def _get_query_embedding(self, query: str): - """Get embedding for the query text""" - embedding_model = self.model_variables['embedding_model'] - query_embedding = embedding_model.embed_query(query) - return query_embedding - - @property - def model_variables(self) -> ModelVariables: - return self._model_variables - - @property - def tenant_info(self) -> Dict[str, Any]: - return self._tenant_info \ No newline at end of file diff --git a/common/langchain/retrievers/eveai_history_retriever.py b/common/langchain/retrievers/eveai_history_retriever.py deleted file mode 100644 index 45875af..0000000 --- a/common/langchain/retrievers/eveai_history_retriever.py +++ /dev/null @@ -1,52 +0,0 @@ -from langchain_core.retrievers import BaseRetriever -from sqlalchemy import asc -from sqlalchemy.exc import SQLAlchemyError -from pydantic import Field, BaseModel, PrivateAttr -from typing import Any, Dict -from flask import current_app - -from common.extensions import db -from common.models.interaction import ChatSession, Interaction -from common.utils.model_utils import ModelVariables - - -class EveAIHistoryRetriever(BaseRetriever, BaseModel): - _model_variables: ModelVariables = PrivateAttr() - _session_id: str = PrivateAttr() - - def __init__(self, model_variables: ModelVariables, session_id: str): - super().__init__() - self._model_variables = model_variables - self._session_id = session_id - - @property - def model_variables(self) -> ModelVariables: - return self._model_variables - - @property - def session_id(self) -> str: - return self._session_id - - def _get_relevant_documents(self, query: str): - current_app.logger.debug(f'Retrieving history of interactions for query: {query}') - - try: - query_obj = ( - db.session.query(Interaction) - .join(ChatSession, Interaction.chat_session_id == ChatSession.id) - .filter(ChatSession.session_id == self.session_id) - .order_by(asc(Interaction.id)) - ) - - interactions = query_obj.all() - - result = [] - for interaction in interactions: - result.append(f'HUMAN:\n{interaction.detailed_question}\n\nAI: \n{interaction.answer}\n\n') - - except SQLAlchemyError as e: - current_app.logger.error(f'Error retrieving history of interactions: {e}') - db.session.rollback() - return [] - - return result \ No newline at end of file diff --git a/common/langchain/retrievers/eveai_retriever.py b/common/langchain/retrievers/eveai_retriever.py deleted file mode 100644 index 3ce57b1..0000000 --- a/common/langchain/retrievers/eveai_retriever.py +++ /dev/null @@ -1,40 +0,0 @@ -from pydantic import BaseModel, PrivateAttr -from typing import Dict, Any - -from common.utils.model_utils import ModelVariables - - -class EveAIRetriever(BaseModel): - _catalog_id: int = PrivateAttr() - _user_metadata: Dict[str, Any] = PrivateAttr() - _system_metadata: Dict[str, Any] = PrivateAttr() - _configuration: Dict[str, Any] = PrivateAttr() - _tenant_info: Dict[str, Any] = PrivateAttr() - _model_variables: ModelVariables = PrivateAttr() - _tuning: bool = PrivateAttr() - - def __init__(self, catalog_id: int, user_metadata: Dict[str, Any], system_metadata: Dict[str, Any], - configuration: Dict[str, Any]): - super().__init__() - self._catalog_id = catalog_id - self._user_metadata = user_metadata - self._system_metadata = system_metadata - self._configuration = configuration - - @property - def catalog_id(self): - return self._catalog_id - - @property - def user_metadata(self): - return self._user_metadata - - @property - def system_metadata(self): - return self._system_metadata - - @property - def configuration(self): - return self._configuration - - # Any common methods that should be shared among retrievers can go here. diff --git a/common/langchain/templates/template_manager.py b/common/langchain/templates/template_manager.py new file mode 100644 index 0000000..5dd8d67 --- /dev/null +++ b/common/langchain/templates/template_manager.py @@ -0,0 +1,154 @@ +import os +import yaml +from typing import Dict, Optional, Any +from packaging import version +from dataclasses import dataclass +from flask import current_app, Flask + +from common.utils.os_utils import get_project_root + + +@dataclass +class PromptTemplate: + """Represents a versioned prompt template""" + content: str + version: str + metadata: Dict[str, Any] + + +class TemplateManager: + """Manages versioned prompt templates""" + + def __init__(self): + self.templates_dir = None + self._templates = None + self.app = None + + def init_app(self, app: Flask) -> None: + # Initialize template manager + base_dir = "/app" + self.templates_dir = os.path.join(base_dir, 'config', 'prompts') + app.logger.debug(f'Loading templates from {self.templates_dir}') + self.app = app + self._templates = self._load_templates() + # Log available templates for each supported model + for llm in app.config['SUPPORTED_LLMS']: + try: + available_templates = self.list_templates(llm) + app.logger.info(f"Loaded templates for {llm}: {available_templates}") + except ValueError: + app.logger.warning(f"No templates found for {llm}") + + def _load_templates(self) -> Dict[str, Dict[str, Dict[str, PromptTemplate]]]: + """ + Load all template versions from the templates directory. + Structure: {provider.model -> {template_name -> {version -> template}}} + Directory structure: + prompts/ + ├── provider/ + │ └── model/ + │ └── template_name/ + │ └── version.yaml + """ + templates = {} + + # Iterate through providers (anthropic, openai) + for provider in os.listdir(self.templates_dir): + provider_path = os.path.join(self.templates_dir, provider) + if not os.path.isdir(provider_path): + continue + + # Iterate through models (claude-3, gpt-4o) + for model in os.listdir(provider_path): + model_path = os.path.join(provider_path, model) + if not os.path.isdir(model_path): + continue + + provider_model = f"{provider}.{model}" + templates[provider_model] = {} + + # Iterate through template types (rag, summary, etc.) + for template_name in os.listdir(model_path): + template_path = os.path.join(model_path, template_name) + if not os.path.isdir(template_path): + continue + + template_versions = {} + # Load all version files for this template + for version_file in os.listdir(template_path): + if not version_file.endswith('.yaml'): + continue + + version_str = version_file[:-5] # Remove .yaml + if not self._is_valid_version(version_str): + current_app.logger.warning( + f"Invalid version format for {template_name}: {version_str}") + continue + + try: + with open(os.path.join(template_path, version_file)) as f: + template_data = yaml.safe_load(f) + # Verify required fields + if not template_data.get('content'): + raise ValueError("Template content is required") + + template_versions[version_str] = PromptTemplate( + content=template_data['content'], + version=version_str, + metadata=template_data.get('metadata', {}) + ) + except Exception as e: + current_app.logger.error( + f"Error loading template {template_name} version {version_str}: {e}") + continue + + if template_versions: + templates[provider_model][template_name] = template_versions + + return templates + + def _is_valid_version(self, version_str: str) -> bool: + """Validate semantic versioning string""" + try: + version.parse(version_str) + return True + except version.InvalidVersion: + return False + + def get_template(self, + provider_model: str, + template_name: str, + template_version: Optional[str] = None) -> PromptTemplate: + """ + Get a specific template version. If version not specified, + returns the latest version. + """ + if provider_model not in self._templates: + raise ValueError(f"Unknown provider.model: {provider_model}") + + if template_name not in self._templates[provider_model]: + raise ValueError(f"Unknown template: {template_name}") + + versions = self._templates[provider_model][template_name] + + if template_version: + if template_version not in versions: + raise ValueError(f"Template version {template_version} not found") + return versions[template_version] + + # Return latest version + latest = max(versions.keys(), key=version.parse) + return versions[latest] + + def list_templates(self, provider_model: str) -> Dict[str, list]: + """ + List all available templates and their versions for a provider.model + Returns: {template_name: [version1, version2, ...]} + """ + if provider_model not in self._templates: + raise ValueError(f"Unknown provider.model: {provider_model}") + + return { + template_name: sorted(versions.keys(), key=version.parse) + for template_name, versions in self._templates[provider_model].items() + } diff --git a/common/langchain/tracked_transcribe.py b/common/langchain/tracked_transcribe.py deleted file mode 100644 index 1667a5b..0000000 --- a/common/langchain/tracked_transcribe.py +++ /dev/null @@ -1,27 +0,0 @@ -import time -from common.utils.business_event_context import current_event - - -def tracked_transcribe(client, *args, **kwargs): - start_time = time.time() - - # Extract the file and model from kwargs if present, otherwise use defaults - file = kwargs.get('file') - model = kwargs.get('model', 'whisper-1') - duration = kwargs.pop('duration', 600) - - result = client.audio.transcriptions.create(*args, **kwargs) - end_time = time.time() - - # Token usage for transcriptions is actually the duration in seconds we pass, as the whisper model is priced per second transcribed - - metrics = { - 'total_tokens': duration, - 'prompt_tokens': 0, # For transcriptions, all tokens are considered "completion" - 'completion_tokens': duration, - 'time_elapsed': end_time - start_time, - 'interaction_type': 'ASR', - } - current_event.log_llm_metrics(metrics) - - return result diff --git a/common/langchain/tracked_transcription.py b/common/langchain/tracked_transcription.py new file mode 100644 index 0000000..8570396 --- /dev/null +++ b/common/langchain/tracked_transcription.py @@ -0,0 +1,77 @@ +# common/langchain/tracked_transcription.py +from typing import Any, Optional, Dict +import time +from openai import OpenAI +from common.utils.business_event_context import current_event + + +class TrackedOpenAITranscription: + """Wrapper for OpenAI transcription with metric tracking""" + + def __init__(self, api_key: str, **kwargs: Any): + """Initialize with OpenAI client settings""" + self.client = OpenAI(api_key=api_key) + self.model = kwargs.get('model', 'whisper-1') + + def transcribe(self, + file: Any, + model: Optional[str] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + response_format: Optional[str] = None, + temperature: Optional[float] = None, + duration: Optional[int] = None) -> str: + """ + Transcribe audio with metrics tracking + + Args: + file: Audio file to transcribe + model: Model to use (defaults to whisper-1) + language: Optional language of the audio + prompt: Optional prompt to guide transcription + response_format: Response format (json, text, etc) + temperature: Sampling temperature + duration: Duration of audio in seconds for metrics + + Returns: + Transcription text + """ + start_time = time.time() + + try: + # Create transcription options + options = { + "file": file, + "model": model or self.model, + } + if language: + options["language"] = language + if prompt: + options["prompt"] = prompt + if response_format: + options["response_format"] = response_format + if temperature: + options["temperature"] = temperature + + response = self.client.audio.transcriptions.create(**options) + + # Calculate metrics + end_time = time.time() + + # Token usage for transcriptions is based on audio duration + metrics = { + 'total_tokens': duration or 600, # Default to 10 minutes if duration not provided + 'prompt_tokens': 0, # For transcriptions, all tokens are completion + 'completion_tokens': duration or 600, + 'time_elapsed': end_time - start_time, + 'interaction_type': 'ASR', + } + current_event.log_llm_metrics(metrics) + + # Return text from response + if isinstance(response, str): + return response + return response.text + + except Exception as e: + raise Exception(f"Transcription failed: {str(e)}") \ No newline at end of file diff --git a/common/models/document.py b/common/models/document.py index 89f6edb..a54bc7c 100644 --- a/common/models/document.py +++ b/common/models/document.py @@ -12,22 +12,31 @@ class Catalog(db.Model): description = db.Column(db.Text, nullable=True) type = db.Column(db.String(50), nullable=False, default="STANDARD_CATALOG") - # Embedding variables - html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']) - html_end_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'li']) - html_included_elements = db.Column(ARRAY(sa.String(50)), nullable=True) - html_excluded_elements = db.Column(ARRAY(sa.String(50)), nullable=True) - html_excluded_classes = db.Column(ARRAY(sa.String(200)), nullable=True) - min_chunk_size = db.Column(db.Integer, nullable=True, default=2000) max_chunk_size = db.Column(db.Integer, nullable=True, default=3000) - # Chat variables ==> Move to Specialist? - chat_RAG_temperature = db.Column(db.Float, nullable=True, default=0.3) - chat_no_RAG_temperature = db.Column(db.Float, nullable=True, default=0.5) + # Meta Data + user_metadata = db.Column(JSONB, nullable=True) + system_metadata = db.Column(JSONB, nullable=True) + configuration = db.Column(JSONB, nullable=True) + + # Versioning Information + created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now()) + created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now()) + updated_by = db.Column(db.Integer, db.ForeignKey(User.id)) + + +class Processor(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(50), nullable=False) + description = db.Column(db.Text, nullable=True) + catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True) + type = db.Column(db.String(50), nullable=False) + sub_file_type = db.Column(db.String(50), nullable=True) # Tuning enablers - embed_tuning = db.Column(db.Boolean, nullable=True, default=False) + tuning = db.Column(db.Boolean, nullable=True, default=False) # Meta Data user_metadata = db.Column(JSONB, nullable=True) @@ -90,6 +99,7 @@ class DocumentVersion(db.Model): bucket_name = db.Column(db.String(255), nullable=True) object_name = db.Column(db.String(200), nullable=True) file_type = db.Column(db.String(20), nullable=True) + sub_file_type = db.Column(db.String(50), nullable=True) file_size = db.Column(db.Float, nullable=True) language = db.Column(db.String(2), nullable=False) user_context = db.Column(db.Text, nullable=True) diff --git a/common/models/interaction.py b/common/models/interaction.py index c6f076c..3eae9e8 100644 --- a/common/models/interaction.py +++ b/common/models/interaction.py @@ -20,34 +20,6 @@ class ChatSession(db.Model): return f"" -class Interaction(db.Model): - id = db.Column(db.Integer, primary_key=True) - chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=False) - question = db.Column(db.Text, nullable=False) - detailed_question = db.Column(db.Text, nullable=True) - answer = db.Column(db.Text, nullable=True) - algorithm_used = db.Column(db.String(20), nullable=True) - language = db.Column(db.String(2), nullable=False) - timezone = db.Column(db.String(30), nullable=True) - appreciation = db.Column(db.Integer, nullable=True) - - # Timing information - question_at = db.Column(db.DateTime, nullable=False) - detailed_question_at = db.Column(db.DateTime, nullable=True) - answer_at = db.Column(db.DateTime, nullable=True) - - # Relations - embeddings = db.relationship('InteractionEmbedding', backref='interaction', lazy=True) - - def __repr__(self): - return f"" - - -class InteractionEmbedding(db.Model): - interaction_id = db.Column(db.Integer, db.ForeignKey(Interaction.id, ondelete='CASCADE'), primary_key=True) - embedding_id = db.Column(db.Integer, db.ForeignKey(Embedding.id, ondelete='CASCADE'), primary_key=True) - - class Specialist(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(50), nullable=False) @@ -68,7 +40,34 @@ class Specialist(db.Model): updated_by = db.Column(db.Integer, db.ForeignKey(User.id)) +class Interaction(db.Model): + id = db.Column(db.Integer, primary_key=True) + chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=False) + specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=True) + specialist_arguments = db.Column(JSONB, nullable=True) + specialist_results = db.Column(JSONB, nullable=True) + timezone = db.Column(db.String(30), nullable=True) + appreciation = db.Column(db.Integer, nullable=True) + + # Timing information + question_at = db.Column(db.DateTime, nullable=False) + detailed_question_at = db.Column(db.DateTime, nullable=True) + answer_at = db.Column(db.DateTime, nullable=True) + + # Relations + embeddings = db.relationship('InteractionEmbedding', backref='interaction', lazy=True) + + def __repr__(self): + return f"" + + +class InteractionEmbedding(db.Model): + interaction_id = db.Column(db.Integer, db.ForeignKey(Interaction.id, ondelete='CASCADE'), primary_key=True) + embedding_id = db.Column(db.Integer, db.ForeignKey(Embedding.id, ondelete='CASCADE'), primary_key=True) + + class SpecialistRetriever(db.Model): specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True) retriever_id = db.Column(db.Integer, db.ForeignKey(Retriever.id, ondelete='CASCADE'), primary_key=True) + retriever = db.relationship("Retriever", backref="specialist_retrievers") diff --git a/common/utils/business_event.py b/common/utils/business_event.py index 654992e..c238c66 100644 --- a/common/utils/business_event.py +++ b/common/utils/business_event.py @@ -4,7 +4,6 @@ from contextlib import contextmanager from datetime import datetime from typing import Dict, Any, Optional from datetime import datetime as dt, timezone as tz -from portkey_ai import Portkey, Config import logging from .business_event_context import BusinessEventContext diff --git a/common/langchain/retrievers/__init__.py b/common/utils/cache/__init__old.py similarity index 100% rename from common/langchain/retrievers/__init__.py rename to common/utils/cache/__init__old.py diff --git a/common/utils/cache/base.py b/common/utils/cache/base.py new file mode 100644 index 0000000..80069ee --- /dev/null +++ b/common/utils/cache/base.py @@ -0,0 +1,89 @@ +# common/utils/cache/base.py + +from typing import Any, Dict, List, Optional, TypeVar, Generic, Type +from dataclasses import dataclass +from flask import Flask +from dogpile.cache import CacheRegion + +T = TypeVar('T') + + +@dataclass +class CacheKey: + """Represents a cache key with multiple components""" + components: Dict[str, Any] + + def __str__(self) -> str: + return ":".join(f"{k}={v}" for k, v in sorted(self.components.items())) + + +class CacheInvalidationManager: + """Manages cache invalidation subscriptions""" + + def __init__(self): + self._subscribers = {} + + def subscribe(self, model: str, handler: 'CacheHandler', key_fields: List[str]): + if model not in self._subscribers: + self._subscribers[model] = [] + self._subscribers[model].append((handler, key_fields)) + + def notify_change(self, model: str, **identifiers): + if model in self._subscribers: + for handler, key_fields in self._subscribers[model]: + if all(field in identifiers for field in key_fields): + handler.invalidate_by_model(model, **identifiers) + + +class CacheHandler(Generic[T]): + """Base cache handler implementation""" + + def __init__(self, region: CacheRegion, prefix: str): + self.region = region + self.prefix = prefix + self._key_components = [] + + def configure_keys(self, *components: str): + self._key_components = components + return self + + def subscribe_to_model(self, model: str, key_fields: List[str]): + invalidation_manager.subscribe(model, self, key_fields) + return self + + def generate_key(self, **identifiers) -> str: + missing = set(self._key_components) - set(identifiers.keys()) + if missing: + raise ValueError(f"Missing key components: {missing}") + + key = CacheKey({k: identifiers[k] for k in self._key_components}) + return f"{self.prefix}:{str(key)}" + + def get(self, creator_func, **identifiers) -> T: + cache_key = self.generate_key(**identifiers) + + def creator(): + instance = creator_func(**identifiers) + return self.to_cache_data(instance) + + cached_data = self.region.get_or_create( + cache_key, + creator, + should_cache_fn=self.should_cache + ) + + return self.from_cache_data(cached_data, **identifiers) + + def invalidate(self, **identifiers): + cache_key = self.generate_key(**identifiers) + self.region.delete(cache_key) + + def invalidate_by_model(self, model: str, **identifiers): + try: + self.invalidate(**identifiers) + except ValueError: + pass + + +# Create global invalidation manager +invalidation_manager = CacheInvalidationManager() \ No newline at end of file diff --git a/common/utils/cache/eveai_cache_manager.py b/common/utils/cache/eveai_cache_manager.py new file mode 100644 index 0000000..fe68367 --- /dev/null +++ b/common/utils/cache/eveai_cache_manager.py @@ -0,0 +1,32 @@ +from typing import Type + +from flask import Flask + +from common.utils.cache.base import CacheHandler + + +class EveAICacheManager: + """Cache manager with registration capabilities""" + + def __init__(self): + self.model_region = None + self.eveai_chat_workers_region = None + self.eveai_workers_region = None + self._handlers = {} + + def init_app(self, app: Flask): + """Initialize cache regions""" + from common.utils.cache.regions import create_cache_regions + self.model_region, self.eveai_chat_workers_region, self.eveai_workers_region = create_cache_regions(app) + + # Initialize all registered handlers with their regions + for handler_class, region_name in self._handlers.items(): + region = getattr(self, f"{region_name}_region") + handler_instance = handler_class(region) + setattr(self, handler_class.handler_name, handler_instance) + + def register_handler(self, handler_class: Type[CacheHandler], region: str): + """Register a cache handler class with its region""" + if not hasattr(handler_class, 'handler_name'): + raise ValueError("Cache handler must define handler_name class attribute") + self._handlers[handler_class] = region diff --git a/common/utils/cache/regions.py b/common/utils/cache/regions.py new file mode 100644 index 0000000..7828fd5 --- /dev/null +++ b/common/utils/cache/regions.py @@ -0,0 +1,61 @@ +# common/utils/cache/regions.py + +from dogpile.cache import make_region +from flask import current_app +from urllib.parse import urlparse +import os + + +def get_redis_config(app): + """ + Create Redis configuration dict based on app config + Handles both authenticated and non-authenticated setups + """ + # Parse the REDIS_BASE_URI to get all components + redis_uri = urlparse(app.config['REDIS_BASE_URI']) + + config = { + 'host': redis_uri.hostname, + 'port': int(redis_uri.port or 6379), + 'db': 4, # Keep this for later use + 'redis_expiration_time': 3600, + 'distributed_lock': True + } + + # Add authentication if provided + if redis_uri.username and redis_uri.password: + config.update({ + 'username': redis_uri.username, + 'password': redis_uri.password + }) + + return config + + +def create_cache_regions(app): + """Initialize all cache regions with app config""" + redis_config = get_redis_config(app) + + # Region for model-related caching (ModelVariables etc) + model_region = make_region(name='model').configure( + 'dogpile.cache.redis', + arguments=redis_config, + replace_existing_backend=True + ) + + # Region for eveai_chat_workers components (Specialists, Retrievers, ...) + eveai_chat_workers_region = make_region(name='chat_workers').configure( + 'dogpile.cache.redis', + arguments=redis_config, # arguments={**redis_config, 'db': 4}, # Different DB + replace_existing_backend=True + ) + + # Region for eveai_workers components (Processors, ...) + eveai_workers_region = make_region(name='workers').configure( + 'dogpile.cache.redis', + arguments=redis_config, # Same config for now + replace_existing_backend=True + ) + + return model_region, eveai_chat_workers_region, eveai_workers_region + diff --git a/common/utils/celery_utils.py b/common/utils/celery_utils.py index 6b2ef0b..47ee9d9 100644 --- a/common/utils/celery_utils.py +++ b/common/utils/celery_utils.py @@ -8,8 +8,6 @@ celery_app = Celery() def init_celery(celery, app, is_beat=False): celery_app.main = app.name - app.logger.debug(f'CELERY_BROKER_URL: {app.config["CELERY_BROKER_URL"]}') - app.logger.debug(f'CELERY_RESULT_BACKEND: {app.config["CELERY_RESULT_BACKEND"]}') celery_config = { 'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'), diff --git a/common/utils/config_field_types.py b/common/utils/config_field_types.py new file mode 100644 index 0000000..6346a10 --- /dev/null +++ b/common/utils/config_field_types.py @@ -0,0 +1,613 @@ +from typing import Optional, List, Union, Dict, Any, Pattern +from pydantic import BaseModel, field_validator, model_validator +from typing_extensions import Annotated +import re +from datetime import datetime +import json +from textwrap import dedent +import yaml +from dataclasses import dataclass + + +class TaggingField(BaseModel): + """Represents a single tagging field configuration""" + type: str + required: bool = False + description: Optional[str] = None + allowed_values: Optional[List[Any]] = None # for enum type + min_value: Optional[Union[int, float]] = None # for numeric types + max_value: Optional[Union[int, float]] = None # for numeric types + + @field_validator('type', mode='before') + @classmethod + def validate_type(cls, v: str) -> str: + valid_types = ['string', 'integer', 'float', 'date', 'enum'] + if v not in valid_types: + raise ValueError(f'type must be one of {valid_types}') + return v + + @model_validator(mode='after') + def validate_field_constraints(self) -> 'TaggingField': + # Validate enum constraints + if self.type == 'enum': + if not self.allowed_values: + raise ValueError('allowed_values must be provided for enum type') + elif self.allowed_values is not None: + raise ValueError('allowed_values only valid for enum type') + + # Validate numeric constraints + if self.type not in ('integer', 'float'): + if self.min_value is not None or self.max_value is not None: + raise ValueError('min_value/max_value only valid for numeric types') + else: + if self.min_value is not None and self.max_value is not None and self.min_value >= self.max_value: + raise ValueError('min_value must be less than max_value') + + return self + + +class TaggingFields(BaseModel): + """Represents a collection of tagging fields, mapped by their names""" + fields: Dict[str, TaggingField] + + @classmethod + def from_dict(cls, data: Dict[str, Dict[str, Any]]) -> 'TaggingFields': + return cls(fields={ + field_name: TaggingField(**field_config) + for field_name, field_config in data.items() + }) + + def to_dict(self) -> Dict[str, Dict[str, Any]]: + return { + field_name: field.model_dump(exclude_none=True) + for field_name, field in self.fields.items() + } + + +class ArgumentConstraint(BaseModel): + """Base class for all argument constraints""" + description: Optional[str] = None + error_message: Optional[str] = None + + +class NumericConstraint(ArgumentConstraint): + """Constraints for numeric values (int/float)""" + min_value: Optional[float] = None + max_value: Optional[float] = None + include_min: bool = True # True for >= min_value, False for > min_value + include_max: bool = True # True for <= max_value, False for < max_value + + @model_validator(mode='after') + def validate_ranges(self) -> 'NumericConstraint': + if self.min_value is not None and self.max_value is not None: + if self.min_value > self.max_value: + raise ValueError("min_value must be less than or equal to max_value") + return self + + def validate(self, value: Union[int, float]) -> bool: + if self.min_value is not None: + if self.include_min and value < self.min_value: + return False + if not self.include_min and value <= self.min_value: + return False + if self.max_value is not None: + if self.include_max and value > self.max_value: + return False + if not self.include_max and value >= self.max_value: + return False + return True + + +class StringConstraint(ArgumentConstraint): + """Constraints for string values""" + min_length: Optional[int] = None + max_length: Optional[int] = None + patterns: Optional[List[str]] = None # List of regex patterns to match + pattern_match_all: bool = False # If True, string must match all patterns + forbidden_patterns: Optional[List[str]] = None # List of regex patterns that must not match + allow_empty: bool = False + + @field_validator('patterns', 'forbidden_patterns') + @classmethod + def validate_patterns(cls, v: Optional[List[str]]) -> Optional[List[str]]: + if v is not None: + # Validate each pattern compiles + for pattern in v: + try: + re.compile(pattern) + except re.error as e: + raise ValueError(f"Invalid regex pattern '{pattern}': {str(e)}") + return v + + def validate(self, value: str) -> bool: + if not self.allow_empty and not value: + return False + + if self.min_length is not None and len(value) < self.min_length: + return False + + if self.max_length is not None and len(value) > self.max_length: + return False + + if self.patterns: + matches = [bool(re.search(pattern, value)) for pattern in self.patterns] + if self.pattern_match_all and not all(matches): + return False + if not self.pattern_match_all and not any(matches): + return False + + if self.forbidden_patterns: + for pattern in self.forbidden_patterns: + if re.search(pattern, value): + return False + + return True + + +class DateConstraint(ArgumentConstraint): + """Constraints for date values""" + min_date: Optional[datetime] = None + max_date: Optional[datetime] = None + include_min: bool = True + include_max: bool = True + allowed_formats: Optional[List[str]] = None # List of allowed date formats + + @model_validator(mode='after') + def validate_ranges(self) -> 'DateConstraint': + if self.min_date and self.max_date and self.min_date > self.max_date: + raise ValueError("min_date must be less than or equal to max_date") + return self + + def validate(self, value: datetime) -> bool: + if self.min_date is not None: + if self.include_min and value < self.min_date: + return False + if not self.include_min and value <= self.min_date: + return False + + if self.max_date is not None: + if self.include_max and value > self.max_date: + return False + if not self.include_max and value >= self.max_date: + return False + + return True + + +class EnumConstraint(ArgumentConstraint): + """Constraints for enum values""" + allowed_values: List[Any] + case_sensitive: bool = True # For string enums + allow_multiple: bool = False # If True, value can be a list of allowed values + min_selections: Optional[int] = None # When allow_multiple is True + max_selections: Optional[int] = None # When allow_multiple is True + + @model_validator(mode='after') + def validate_selections(self) -> 'EnumConstraint': + if self.allow_multiple: + if self.min_selections is not None and self.max_selections is not None: + if self.min_selections > self.max_selections: + raise ValueError("min_selections must be less than or equal to max_selections") + if self.max_selections > len(self.allowed_values): + raise ValueError("max_selections cannot be greater than number of allowed values") + return self + + def validate(self, value: Union[Any, List[Any]]) -> bool: + if self.allow_multiple: + if not isinstance(value, list): + return False + + if self.min_selections is not None and len(value) < self.min_selections: + return False + + if self.max_selections is not None and len(value) > self.max_selections: + return False + + for v in value: + if not self._validate_single_value(v): + return False + else: + return self._validate_single_value(value) + + return True + + def _validate_single_value(self, value: Any) -> bool: + if isinstance(value, str) and not self.case_sensitive: + return any(str(value).lower() == str(v).lower() for v in self.allowed_values) + return value in self.allowed_values + + +class ArgumentDefinition(BaseModel): + """Defines an argument with its type and constraints""" + name: str + type: str + description: Optional[str] = None + required: bool = False + default: Optional[Any] = None + constraints: Optional[Union[NumericConstraint, StringConstraint, DateConstraint, EnumConstraint]] = None + + @field_validator('type') + @classmethod + def validate_type(cls, v: str) -> str: + valid_types = ['string', 'integer', 'float', 'date', 'enum'] + if v not in valid_types: + raise ValueError(f'type must be one of {valid_types}') + return v + + @model_validator(mode='after') + def validate_constraints(self) -> 'ArgumentDefinition': + if self.constraints: + expected_constraint_types = { + 'string': StringConstraint, + 'integer': NumericConstraint, + 'float': NumericConstraint, + 'date': DateConstraint, + 'enum': EnumConstraint + } + + expected_type = expected_constraint_types.get(self.type) + if not isinstance(self.constraints, expected_type): + raise ValueError(f'Constraints for type {self.type} must be of type {expected_type.__name__}') + + if self.default is not None: + if not self.constraints.validate(self.default): + raise ValueError(f'Default value does not satisfy constraints for {self.name}') + + return self + + +class ArgumentDefinitions(BaseModel): + """Collection of argument definitions""" + arguments: Dict[str, ArgumentDefinition] + + @classmethod + def from_dict(cls, data: Dict[str, Dict[str, Any]]) -> 'ArgumentDefinitions': + return cls(arguments={ + arg_name: ArgumentDefinition(**arg_config) + for arg_name, arg_config in data.items() + }) + + def to_dict(self) -> Dict[str, Dict[str, Any]]: + return { + arg_name: arg.model_dump(exclude_none=True) + for arg_name, arg in self.arguments.items() + } + + def validate_argument_values(self, values: Dict[str, Any]) -> Dict[str, str]: + """ + Validate a set of argument values against their definitions + Returns a dictionary of error messages for invalid arguments + """ + errors = {} + + # Check for required arguments + for name, arg_def in self.arguments.items(): + if arg_def.required and name not in values: + errors[name] = "Required argument missing" + continue + + if name in values: + value = values[name] + + # Validate type + try: + if arg_def.type == 'integer': + value = int(value) + elif arg_def.type == 'float': + value = float(value) + elif arg_def.type == 'date' and isinstance(value, str): + if arg_def.constraints and arg_def.constraints.allowed_formats: + for fmt in arg_def.constraints.allowed_formats: + try: + value = datetime.strptime(value, fmt) + break + except ValueError: + continue + else: + errors[ + name] = f"Invalid date format. Allowed formats: {arg_def.constraints.allowed_formats}" + continue + except (ValueError, TypeError): + errors[name] = f"Invalid type. Expected {arg_def.type}" + continue + + # Validate constraints + if arg_def.constraints and not arg_def.constraints.validate(value): + errors[name] = arg_def.constraints.error_message or "Value does not satisfy constraints" + + return errors + + +@dataclass +class DocumentationFormat: + """Constants for documentation formats""" + MARKDOWN = "markdown" + JSON = "json" + YAML = "yaml" + + +@dataclass +class DocumentationVersion: + """Constants for documentation versions""" + BASIC = "basic" # Original documentation without retriever info + EXTENDED = "extended" # Including retriever documentation + + +def _generate_argument_constraints(field_config: Dict[str, Any]) -> List[Dict[str, Any]]: + """Generate possible argument constraints based on field type""" + constraints = [] + + base_constraint = { + "description": f"Constraint for {field_config.get('description', 'field')}", + "error_message": "Optional custom error message" + } + + if field_config["type"] == "integer" or field_config["type"] == "float": + constraints.append({ + **base_constraint, + "type": "NumericConstraint", + "possible_constraints": { + "min_value": "number", + "max_value": "number", + "include_min": "boolean", + "include_max": "boolean" + }, + "example": { + "min_value": field_config.get("min_value", 0), + "max_value": field_config.get("max_value", 100), + "include_min": True, + "include_max": True + } + }) + + elif field_config["type"] == "string": + constraints.append({ + **base_constraint, + "type": "StringConstraint", + "possible_constraints": { + "min_length": "integer", + "max_length": "integer", + "patterns": "list[str]", + "pattern_match_all": "boolean", + "forbidden_patterns": "list[str]", + "allow_empty": "boolean" + }, + "example": { + "min_length": 1, + "max_length": 100, + "patterns": ["^[A-Za-z0-9]+$"], + "pattern_match_all": False, + "forbidden_patterns": ["^test_", "_temp$"], + "allow_empty": False + } + }) + + elif field_config["type"] == "enum": + constraints.append({ + **base_constraint, + "type": "EnumConstraint", + "possible_constraints": { + "allowed_values": f"list[{field_config.get('allowed_values', ['value1', 'value2'])}]", + "case_sensitive": "boolean", + "allow_multiple": "boolean", + "min_selections": "integer", + "max_selections": "integer" + }, + "example": { + "allowed_values": field_config.get("allowed_values", ["value1", "value2"]), + "case_sensitive": True, + "allow_multiple": True, + "min_selections": 1, + "max_selections": 2 + } + }) + + elif field_config["type"] == "date": + constraints.append({ + **base_constraint, + "type": "DateConstraint", + "possible_constraints": { + "min_date": "datetime", + "max_date": "datetime", + "include_min": "boolean", + "include_max": "boolean", + "allowed_formats": "list[str]" + }, + "example": { + "min_date": "2024-01-01T00:00:00", + "max_date": "2024-12-31T23:59:59", + "include_min": True, + "include_max": True, + "allowed_formats": ["%Y-%m-%d", "%Y/%m/%d"] + } + }) + + return constraints + + +def generate_field_documentation( + tagging_fields: Dict[str, Any], + format: str = "markdown", + version: str = "basic" +) -> str: + """ + Generate documentation for tagging fields configuration. + + Args: + tagging_fields: Dictionary containing tagging fields configuration + format: Output format ("markdown", "json", or "yaml") + version: Documentation version ("basic" or "extended") + + Returns: + str: Formatted documentation + """ + if version not in [DocumentationVersion.BASIC, DocumentationVersion.EXTENDED]: + raise ValueError(f"Unsupported documentation version: {version}") + + # Normalize fields configuration + normalized_fields = {} + + for field_name, field_config in tagging_fields.items(): + field_doc = { + "name": field_name, + "type": field_config["type"], + "required": field_config.get("required", False), + "description": field_config.get("description", "No description provided"), + "constraints": [] + } + + # Only include possible arguments in extended version + if version == DocumentationVersion.EXTENDED: + field_doc["possible_arguments"] = _generate_argument_constraints(field_config) + + # Add type-specific constraints + if field_config["type"] == "integer" or field_config["type"] == "float": + if "min_value" in field_config: + field_doc["constraints"].append( + f"Minimum value: {field_config['min_value']}") + if "max_value" in field_config: + field_doc["constraints"].append( + f"Maximum value: {field_config['max_value']}") + + elif field_config["type"] == "string": + if "min_length" in field_config: + field_doc["constraints"].append( + f"Minimum length: {field_config['min_length']}") + if "max_length" in field_config: + field_doc["constraints"].append( + f"Maximum length: {field_config['max_length']}") + if "patterns" in field_config: + field_doc["constraints"].append( + f"Must match patterns: {', '.join(field_config['patterns'])}") + + elif field_config["type"] == "enum": + if "allowed_values" in field_config: + field_doc["constraints"].append( + f"Allowed values: {', '.join(str(v) for v in field_config['allowed_values'])}") + + elif field_config["type"] == "date": + if "min_date" in field_config: + field_doc["constraints"].append( + f"Minimum date: {field_config['min_date']}") + if "max_date" in field_config: + field_doc["constraints"].append( + f"Maximum date: {field_config['max_date']}") + if "allowed_formats" in field_config: + field_doc["constraints"].append( + f"Allowed formats: {', '.join(field_config['allowed_formats'])}") + + normalized_fields[field_name] = field_doc + + # Generate documentation in requested format + if format == DocumentationFormat.MARKDOWN: + return _generate_markdown_docs(normalized_fields, version) + elif format == DocumentationFormat.JSON: + return _generate_json_docs(normalized_fields, version) + elif format == DocumentationFormat.YAML: + return _generate_yaml_docs(normalized_fields, version) + else: + raise ValueError(f"Unsupported documentation format: {format}") + + +def _generate_markdown_docs(fields: Dict[str, Any], version: str) -> str: + """Generate markdown documentation""" + docs = ["# Tagging Fields Documentation\n"] + + # Add overview table + docs.append("## Fields Overview\n") + docs.append("| Field Name | Type | Required | Description |") + docs.append("|------------|------|----------|-------------|") + + for field_name, field in fields.items(): + docs.append( + f"| {field_name} | {field['type']} | " + f"{'Yes' if field['required'] else 'No'} | {field['description']} |" + ) + + # Add detailed field specifications + docs.append("\n## Detailed Field Specifications\n") + + for field_name, field in fields.items(): + docs.append(f"### {field_name}\n") + docs.append(f"**Type:** {field['type']}") + docs.append(f"**Required:** {'Yes' if field['required'] else 'No'}") + docs.append(f"**Description:** {field['description']}\n") + + if field["constraints"]: + docs.append("**Field Constraints:**") + for constraint in field["constraints"]: + docs.append(f"- {constraint}") + docs.append("") + + # Add retriever argument documentation only in extended version + if version == DocumentationVersion.EXTENDED and "possible_arguments" in field: + docs.append("**Possible Retriever Arguments:**") + for arg_constraint in field["possible_arguments"]: + docs.append(f"\n*{arg_constraint['type']}*") + docs.append(f"Description: {arg_constraint['description']}") + docs.append("\nPossible constraints:") + for const_name, const_type in arg_constraint["possible_constraints"].items(): + docs.append(f"- `{const_name}`: {const_type}") + + docs.append("\nExample:") + docs.append("```python") + docs.append(json.dumps(arg_constraint["example"], indent=2)) + docs.append("```\n") + + # Add example retriever configuration only in extended version + if version == DocumentationVersion.EXTENDED: + docs.append("\n## Example Retriever Configuration\n") + docs.append("```python") + example_config = { + "metadata_filters": { + field_name: field["possible_arguments"][0]["example"] + for field_name, field in fields.items() + if "possible_arguments" in field + } + } + docs.append(json.dumps(example_config, indent=2)) + docs.append("```") + + return "\n".join(docs) + + +def _generate_json_docs(fields: Dict[str, Any], version: str) -> str: + """Generate JSON documentation""" + doc = { + "tagging_fields_documentation": { + "version": version, + "fields": fields + } + } + + if version == DocumentationVersion.EXTENDED: + doc["tagging_fields_documentation"]["example_retriever_config"] = { + "metadata_filters": { + field_name: field["possible_arguments"][0]["example"] + for field_name, field in fields.items() + if "possible_arguments" in field + } + } + + return json.dumps(doc, indent=2) + + +def _generate_yaml_docs(fields: Dict[str, Any], version: str) -> str: + """Generate YAML documentation""" + doc = { + "tagging_fields_documentation": { + "version": version, + "fields": fields + } + } + + if version == DocumentationVersion.EXTENDED: + doc["tagging_fields_documentation"]["example_retriever_config"] = { + "metadata_filters": { + field_name: field["possible_arguments"][0]["example"] + for field_name, field in fields.items() + if "possible_arguments" in field + } + } + + return yaml.dump(doc, sort_keys=False, default_flow_style=False) \ No newline at end of file diff --git a/common/utils/cors_utils.py b/common/utils/cors_utils.py index 534a8ba..fc88785 100644 --- a/common/utils/cors_utils.py +++ b/common/utils/cors_utils.py @@ -5,10 +5,8 @@ from common.models.user import Tenant, TenantDomain def get_allowed_origins(tenant_id): session_key = f"allowed_origins_{tenant_id}" if session_key in session: - current_app.logger.debug(f"Fetching allowed origins for tenant {tenant_id} from session") return session[session_key] - current_app.logger.debug(f"Fetching allowed origins for tenant {tenant_id} from database") tenant_domains = TenantDomain.query.filter_by(tenant_id=int(tenant_id)).all() allowed_origins = [domain.domain for domain in tenant_domains] @@ -18,14 +16,8 @@ def get_allowed_origins(tenant_id): def cors_after_request(response, prefix): - current_app.logger.debug(f'CORS after request: {request.path}, prefix: {prefix}') - current_app.logger.debug(f'request.headers: {request.headers}') - current_app.logger.debug(f'request.args: {request.args}') - current_app.logger.debug(f'request is json?: {request.is_json}') - # Exclude health checks from checks if request.path.startswith('/healthz') or request.path.startswith('/_healthz'): - current_app.logger.debug('Skipping CORS headers for health checks') response.headers.add('Access-Control-Allow-Origin', '*') response.headers.add('Access-Control-Allow-Headers', '*') response.headers.add('Access-Control-Allow-Methods', '*') @@ -36,7 +28,6 @@ def cors_after_request(response, prefix): # Try to get tenant_id from JSON payload json_data = request.get_json(silent=True) - current_app.logger.debug(f'request.get_json(silent=True): {json_data}') if json_data and 'tenant_id' in json_data: tenant_id = json_data['tenant_id'] @@ -44,23 +35,17 @@ def cors_after_request(response, prefix): # Fallback to get tenant_id from query parameters or headers if JSON is not available tenant_id = request.args.get('tenant_id') or request.args.get('tenantId') or request.headers.get('X-Tenant-ID') - current_app.logger.debug(f'Identified tenant_id: {tenant_id}') - if tenant_id: allowed_origins = get_allowed_origins(tenant_id) - current_app.logger.debug(f'Allowed origins for tenant {tenant_id}: {allowed_origins}') else: current_app.logger.warning('tenant_id not found in request') origin = request.headers.get('Origin') - current_app.logger.debug(f'Origin: {origin}') - if origin in allowed_origins: response.headers.add('Access-Control-Allow-Origin', origin) response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization') response.headers.add('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS') response.headers.add('Access-Control-Allow-Credentials', 'true') - current_app.logger.debug(f'CORS headers set for origin: {origin}') else: current_app.logger.warning(f'Origin {origin} not allowed') diff --git a/common/utils/debug_utils.py b/common/utils/debug_utils.py index 15d48f7..e965946 100644 --- a/common/utils/debug_utils.py +++ b/common/utils/debug_utils.py @@ -36,7 +36,7 @@ def log_request_middleware(app): @app.before_request def log_session_state_before(): - app.logger.debug(f'Session state before request: {session.items()}') + pass # @app.after_request # def log_response_info(response): @@ -58,5 +58,4 @@ def log_request_middleware(app): @app.after_request def log_session_state_after(response): - app.logger.debug(f'Session state after request: {session.items()}') return response diff --git a/common/utils/document_utils.py b/common/utils/document_utils.py index f2797db..54a80ae 100644 --- a/common/utils/document_utils.py +++ b/common/utils/document_utils.py @@ -24,6 +24,7 @@ def create_document_stack(api_input, file, filename, extension, tenant_id): # Create the DocumentVersion new_doc_vers = create_version_for_document(new_doc, tenant_id, api_input.get('url', ''), + api_input.get('sub_file_type', ''), api_input.get('language', 'en'), api_input.get('user_context', ''), api_input.get('user_metadata'), @@ -64,7 +65,7 @@ def create_document(form, filename, catalog_id): return new_doc -def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata, catalog_properties): +def create_version_for_document(document, tenant_id, url, sub_file_type, language, user_context, user_metadata, catalog_properties): new_doc_vers = DocumentVersion() if url != '': new_doc_vers.url = url @@ -83,6 +84,9 @@ def create_version_for_document(document, tenant_id, url, language, user_context if catalog_properties != '' and catalog_properties is not None: new_doc_vers.catalog_properties = catalog_properties + if sub_file_type != '': + new_doc_vers.sub_file_type = sub_file_type + new_doc_vers.document = document set_logging_information(new_doc_vers, dt.now(tz.utc)) @@ -237,8 +241,6 @@ def start_embedding_task(tenant_id, doc_vers_id): def validate_file_type(extension): - current_app.logger.debug(f'Validating file type {extension}') - current_app.logger.debug(f'Supported file types: {current_app.config["SUPPORTED_FILE_TYPES"]}') if extension not in current_app.config['SUPPORTED_FILE_TYPES']: raise EveAIUnsupportedFileType(f"Filetype {extension} is currently not supported. " f"Supported filetypes: {', '.join(current_app.config['SUPPORTED_FILE_TYPES'])}") diff --git a/common/utils/eveai_exceptions.py b/common/utils/eveai_exceptions.py index fcd54a4..b9937e4 100644 --- a/common/utils/eveai_exceptions.py +++ b/common/utils/eveai_exceptions.py @@ -10,6 +10,7 @@ class EveAIException(Exception): def to_dict(self): rv = dict(self.payload or ()) rv['message'] = self.message + rv['error'] = self.__class__.__name__ return rv @@ -41,3 +42,9 @@ class EveAINoLicenseForTenant(EveAIException): super().__init__(message, status_code, payload) +class EveAITenantNotFound(EveAIException): + """Raised when a tenant is not found""" + + def __init__(self, message="Tenant not found", status_code=400, payload=None): + super().__init__(message, status_code, payload) + diff --git a/common/utils/middleware.py b/common/utils/middleware.py index e14a497..4722817 100644 --- a/common/utils/middleware.py +++ b/common/utils/middleware.py @@ -24,9 +24,6 @@ def mw_before_request(): if not tenant_id: raise Exception('Cannot switch schema for tenant: no tenant defined in session') - for role in current_user.roles: - current_app.logger.debug(f'In middleware: User {current_user.email} has role {role.name}') - # user = User.query.get(current_user.id) if current_user.has_role('Super User') or current_user.tenant_id == tenant_id: Database(tenant_id).switch_schema() diff --git a/common/utils/model_utils.py b/common/utils/model_utils.py index aecb12a..ac2f4a0 100644 --- a/common/utils/model_utils.py +++ b/common/utils/model_utils.py @@ -1,249 +1,36 @@ import os +from typing import Dict, Any, Optional import langcodes -from flask import current_app -from langchain_openai import OpenAIEmbeddings, ChatOpenAI -from langchain_anthropic import ChatAnthropic -from langchain_core.pydantic_v1 import BaseModel, Field -from typing import List, Any, Iterator -from collections.abc import MutableMapping -from openai import OpenAI -from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL -from portkey_ai.langchain.portkey_langchain_callback_handler import LangchainCallbackHandler from common.langchain.llm_metrics_handler import LLMMetricsHandler +from common.langchain.templates.template_manager import TemplateManager +from langchain_openai import OpenAIEmbeddings, ChatOpenAI, OpenAI +from langchain_anthropic import ChatAnthropic +from flask import current_app +from datetime import datetime as dt, timezone as tz + from common.langchain.tracked_openai_embeddings import TrackedOpenAIEmbeddings -from common.langchain.tracked_transcribe import tracked_transcribe -from common.models.document import EmbeddingSmallOpenAI, EmbeddingLargeOpenAI, Catalog +from common.langchain.tracked_transcription import TrackedOpenAITranscription from common.models.user import Tenant +from common.utils.cache.base import CacheHandler from config.model_config import MODEL_CONFIG -from common.utils.business_event_context import current_event +from common.extensions import template_manager, cache_manager +from common.models.document import EmbeddingLargeOpenAI, EmbeddingSmallOpenAI +from common.utils.eveai_exceptions import EveAITenantNotFound -class CitedAnswer(BaseModel): - """Default docstring - to be replaced with actual prompt""" +def create_language_template(template: str, language: str) -> str: + """ + Replace language placeholder in template with specified language - answer: str = Field( - ..., - description="The answer to the user question, based on the given sources", - ) - citations: List[int] = Field( - ..., - description="The integer IDs of the SPECIFIC sources that were used to generate the answer" - ) - insufficient_info: bool = Field( - False, # Default value is set to False - description="A boolean indicating wether given sources were sufficient or not to generate the answer" - ) + Args: + template: Template string with {language} placeholder + language: Language code to insert - -def set_language_prompt_template(cls, language_prompt): - cls.__doc__ = language_prompt - - -class ModelVariables(MutableMapping): - def __init__(self, tenant: Tenant, catalog_id=None): - self.tenant = tenant - self.catalog_id = catalog_id - self._variables = self._initialize_variables() - self._embedding_model = None - self._llm = None - self._llm_no_rag = None - self._transcription_client = None - self._prompt_templates = {} - self._embedding_db_model = None - self.llm_metrics_handler = LLMMetricsHandler() - self._transcription_client = None - - def _initialize_variables(self): - variables = {} - - # Get the Catalog if catalog_id is passed - if self.catalog_id: - catalog = Catalog.query.get_or_404(self.catalog_id) - - # We initialize the variables that are available knowing the tenant. - variables['embed_tuning'] = catalog.embed_tuning or False - - # Set HTML Chunking Variables - variables['html_tags'] = catalog.html_tags - variables['html_end_tags'] = catalog.html_end_tags - variables['html_included_elements'] = catalog.html_included_elements - variables['html_excluded_elements'] = catalog.html_excluded_elements - variables['html_excluded_classes'] = catalog.html_excluded_classes - - # Set Chunk Size variables - variables['min_chunk_size'] = catalog.min_chunk_size - variables['max_chunk_size'] = catalog.max_chunk_size - - # Set the RAG Context (will have to change once specialists are defined - variables['rag_context'] = self.tenant.rag_context or " " - # Temporary setting until we have Specialists - variables['rag_tuning'] = False - variables['RAG_temperature'] = 0.3 - variables['no_RAG_temperature'] = 0.5 - variables['k'] = 8 - variables['similarity_threshold'] = 0.4 - - # Set model providers - variables['embedding_provider'], variables['embedding_model'] = self.tenant.embedding_model.rsplit('.', 1) - variables['llm_provider'], variables['llm_model'] = self.tenant.llm_model.rsplit('.', 1) - variables["templates"] = current_app.config['PROMPT_TEMPLATES'][(f"{variables['llm_provider']}." - f"{variables['llm_model']}")] - current_app.logger.info(f"Loaded prompt templates: \n") - current_app.logger.info(f"{variables['templates']}") - - # Set model-specific configurations - model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {}) - variables.update(model_config) - - variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][self.tenant.llm_model] - - if variables['tool_calling_supported']: - variables['cited_answer_cls'] = CitedAnswer - - variables['max_compression_duration'] = current_app.config['MAX_COMPRESSION_DURATION'] - variables['max_transcription_duration'] = current_app.config['MAX_TRANSCRIPTION_DURATION'] - variables['compression_cpu_limit'] = current_app.config['COMPRESSION_CPU_LIMIT'] - variables['compression_process_delay'] = current_app.config['COMPRESSION_PROCESS_DELAY'] - - return variables - - @property - def embedding_model(self): - api_key = os.getenv('OPENAI_API_KEY') - model = self._variables['embedding_model'] - self._embedding_model = TrackedOpenAIEmbeddings(api_key=api_key, - model=model, - ) - self._embedding_db_model = EmbeddingSmallOpenAI \ - if model == 'text-embedding-3-small' \ - else EmbeddingLargeOpenAI - - return self._embedding_model - - @property - def llm(self): - api_key = self.get_api_key_for_llm() - self._llm = ChatOpenAI(api_key=api_key, - model=self._variables['llm_model'], - temperature=self._variables['RAG_temperature'], - callbacks=[self.llm_metrics_handler]) - return self._llm - - @property - def llm_no_rag(self): - api_key = self.get_api_key_for_llm() - self._llm_no_rag = ChatOpenAI(api_key=api_key, - model=self._variables['llm_model'], - temperature=self._variables['RAG_temperature'], - callbacks=[self.llm_metrics_handler]) - return self._llm_no_rag - - def get_api_key_for_llm(self): - if self._variables['llm_provider'] == 'openai': - api_key = os.getenv('OPENAI_API_KEY') - else: # self._variables['llm_provider'] == 'anthropic' - api_key = os.getenv('ANTHROPIC_API_KEY') - - return api_key - - @property - def transcription_client(self): - api_key = os.getenv('OPENAI_API_KEY') - self._transcription_client = OpenAI(api_key=api_key, ) - self._variables['transcription_model'] = 'whisper-1' - return self._transcription_client - - def transcribe(self, *args, **kwargs): - return tracked_transcribe(self._transcription_client, *args, **kwargs) - - @property - def embedding_db_model(self): - if self._embedding_db_model is None: - self._embedding_db_model = self.get_embedding_db_model() - return self._embedding_db_model - - def get_embedding_db_model(self): - current_app.logger.debug("In get_embedding_db_model") - if self._embedding_db_model is None: - self._embedding_db_model = EmbeddingSmallOpenAI \ - if self._variables['embedding_model'] == 'text-embedding-3-small' \ - else EmbeddingLargeOpenAI - current_app.logger.debug(f"Embedding DB Model: {self._embedding_db_model}") - return self._embedding_db_model - - def get_prompt_template(self, template_name: str) -> str: - current_app.logger.info(f"Getting prompt template for {template_name}") - if template_name not in self._prompt_templates: - self._prompt_templates[template_name] = self._load_prompt_template(template_name) - return self._prompt_templates[template_name] - - def _load_prompt_template(self, template_name: str) -> str: - # In the future, this method will make an API call to Portkey - # For now, we'll simulate it with a placeholder implementation - # You can replace this with your current prompt loading logic - return self._variables['templates'][template_name] - - def __getitem__(self, key: str) -> Any: - current_app.logger.debug(f"ModelVariables: Getting {key}") - # Support older template names (suffix = _template) - if key.endswith('_template'): - key = key[:-len('_template')] - current_app.logger.debug(f"ModelVariables: Getting modified {key}") - if key == 'embedding_model': - return self.embedding_model - elif key == 'embedding_db_model': - return self.embedding_db_model - elif key == 'llm': - return self.llm - elif key == 'llm_no_rag': - return self.llm_no_rag - elif key == 'transcription_client': - return self.transcription_client - elif key in self._variables.get('prompt_templates', []): - return self.get_prompt_template(key) - else: - value = self._variables.get(key) - if value is not None: - return value - else: - raise KeyError(f'Variable {key} does not exist in ModelVariables') - - def __setitem__(self, key: str, value: Any) -> None: - self._variables[key] = value - - def __delitem__(self, key: str) -> None: - del self._variables[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._variables) - - def __len__(self): - return len(self._variables) - - def get(self, key: str, default: Any = None) -> Any: - return self.__getitem__(key) or default - - def update(self, **kwargs) -> None: - self._variables.update(kwargs) - - def items(self): - return self._variables.items() - - def keys(self): - return self._variables.keys() - - def values(self): - return self._variables.values() - - -def select_model_variables(tenant, catalog_id=None): - model_variables = ModelVariables(tenant=tenant, catalog_id=catalog_id) - return model_variables - - -def create_language_template(template, language): + Returns: + str: Template with language placeholder replaced + """ try: full_language = langcodes.Language.make(language=language) language_template = template.replace('{language}', full_language.display_name()) @@ -253,5 +40,249 @@ def create_language_template(template, language): return language_template -def replace_variable_in_template(template, variable, value): - return template.replace(variable, value) +def replace_variable_in_template(template: str, variable: str, value: str) -> str: + """ + Replace a variable placeholder in template with specified value + + Args: + template: Template string with variable placeholder + variable: Variable placeholder to replace (e.g. "{tenant_context}") + value: Value to insert + + Returns: + str: Template with variable placeholder replaced + """ + return template.replace(variable, value or "") + + +class ModelVariables: + """Manages model-related variables and configurations""" + + def __init__(self, tenant_id: int, variables: Dict[str, Any] = None): + """ + Initialize ModelVariables with tenant and optional template manager + + Args: + tenant: Tenant instance + template_manager: Optional TemplateManager instance + """ + current_app.logger.info(f'Model variables initialized with tenant {tenant_id} and variables \n{variables}') + self.tenant_id = tenant_id + self._variables = variables if variables is not None else self._initialize_variables() + current_app.logger.info(f'Model _variables initialized to {self._variables}') + self._embedding_model = None + self._embedding_model_class = None + self._llm_instances = {} + self.llm_metrics_handler = LLMMetricsHandler() + self._transcription_model = None + + def _initialize_variables(self) -> Dict[str, Any]: + """Initialize the variables dictionary""" + variables = {} + + tenant = Tenant.query.get(self.tenant_id) + if not tenant: + raise EveAITenantNotFound(f"Tenant {self.tenant_id} not found") + + # Set model providers + variables['embedding_provider'], variables['embedding_model'] = tenant.embedding_model.split('.') + variables['llm_provider'], variables['llm_model'] = tenant.llm_model.split('.') + variables['llm_full_model'] = tenant.llm_model + + # Set model-specific configurations + model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {}) + variables.update(model_config) + + # Additional configurations + variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][tenant.llm_model] + variables['max_compression_duration'] = current_app.config['MAX_COMPRESSION_DURATION'] + variables['max_transcription_duration'] = current_app.config['MAX_TRANSCRIPTION_DURATION'] + variables['compression_cpu_limit'] = current_app.config['COMPRESSION_CPU_LIMIT'] + variables['compression_process_delay'] = current_app.config['COMPRESSION_PROCESS_DELAY'] + + return variables + + @property + def embedding_model(self): + """Get the embedding model instance""" + if self._embedding_model is None: + api_key = os.getenv('OPENAI_API_KEY') + self._embedding_model = TrackedOpenAIEmbeddings( + api_key=api_key, + model=self._variables['embedding_model'] + ) + return self._embedding_model + + @property + def embedding_model_class(self): + """Get the embedding model class""" + if self._embedding_model_class is None: + if self._variables['embedding_model'] == 'text-embedding-3-large': + self._embedding_model_class = EmbeddingLargeOpenAI + else: # text-embedding-3-small + self._embedding_model_class = EmbeddingSmallOpenAI + + return self._embedding_model_class + + @property + def annotation_chunk_length(self): + return self._variables['annotation_chunk_length'] + + @property + def max_compression_duration(self): + return self._variables['max_compression_duration'] + + @property + def max_transcription_duration(self): + return self._variables['max_transcription_duration'] + + @property + def compression_cpu_limit(self): + return self._variables['compression_cpu_limit'] + + @property + def compression_process_delay(self): + return self._variables['compression_process_delay'] + + def get_llm(self, temperature: float = 0.3, **kwargs) -> Any: + """ + Get an LLM instance with specific configuration + + Args: + temperature: The temperature for the LLM + **kwargs: Additional configuration parameters + + Returns: + An instance of the configured LLM + """ + cache_key = f"{temperature}_{hash(frozenset(kwargs.items()))}" + + if cache_key not in self._llm_instances: + provider = self._variables['llm_provider'] + model = self._variables['llm_model'] + + if provider == 'openai': + self._llm_instances[cache_key] = ChatOpenAI( + api_key=os.getenv('OPENAI_API_KEY'), + model=model, + temperature=temperature, + callbacks=[self.llm_metrics_handler], + **kwargs + ) + elif provider == 'anthropic': + self._llm_instances[cache_key] = ChatAnthropic( + api_key=os.getenv('ANTHROPIC_API_KEY'), + model=current_app.config['ANTHROPIC_LLM_VERSIONS'][model], + temperature=temperature, + callbacks=[self.llm_metrics_handler], + **kwargs + ) + else: + raise ValueError(f"Unsupported LLM provider: {provider}") + + return self._llm_instances[cache_key] + + @property + def transcription_model(self) -> TrackedOpenAITranscription: + """Get the transcription model instance""" + if self._transcription_model is None: + api_key = os.getenv('OPENAI_API_KEY') + self._transcription_model = TrackedOpenAITranscription( + api_key=api_key, + model='whisper-1' + ) + return self._transcription_model + + # Remove the old transcription-related methods since they're now handled by TrackedOpenAITranscription + @property + def transcription_client(self): + raise DeprecationWarning("Use transcription_model instead") + + def transcribe(self, *args, **kwargs): + raise DeprecationWarning("Use transcription_model.transcribe() instead") + + def get_template(self, template_name: str, version: Optional[str] = None) -> str: + """ + Get a template for the tenant's configured LLM + + Args: + template_name: Name of the template to retrieve + version: Optional specific version to retrieve + + Returns: + The template content + """ + try: + template = template_manager.get_template( + self._variables['llm_full_model'], + template_name, + version + ) + return template.content + except Exception as e: + current_app.logger.error(f"Error getting template {template_name}: {str(e)}") + # Fall back to old template loading if template_manager fails + if template_name in self._variables.get('templates', {}): + return self._variables['templates'][template_name] + raise + + +class ModelVariablesCacheHandler(CacheHandler[ModelVariables]): + handler_name = 'model_vars_cache' # Used to access handler instance from cache_manager + + def __init__(self, region): + super().__init__(region, 'model_variables') + self.configure_keys('tenant_id') + self.subscribe_to_model('Tenant', ['tenant_id']) + + def to_cache_data(self, instance: ModelVariables) -> Dict[str, Any]: + return { + 'tenant_id': instance.tenant_id, + 'variables': instance._variables, + 'last_updated': dt.now(tz=tz.utc).isoformat() + } + + def from_cache_data(self, data: Dict[str, Any], tenant_id: int, **kwargs) -> ModelVariables: + instance = ModelVariables(tenant_id, data.get('variables')) + return instance + + def should_cache(self, value: Dict[str, Any]) -> bool: + required_fields = {'tenant_id', 'variables'} + return all(field in value for field in required_fields) + + +# Register the handler with the cache manager +cache_manager.register_handler(ModelVariablesCacheHandler, 'model') + + +# Helper function to get cached model variables +def get_model_variables(tenant_id: int) -> ModelVariables: + return cache_manager.model_vars_cache.get( + lambda tenant_id: ModelVariables(tenant_id), # function to create ModelVariables if required + tenant_id=tenant_id + ) + +# Written in a long format, without lambda +# def get_model_variables(tenant_id: int) -> ModelVariables: +# """ +# Get ModelVariables instance, either from cache or newly created +# +# Args: +# tenant_id: The tenant's ID +# +# Returns: +# ModelVariables: Instance with either cached or fresh data +# +# Raises: +# TenantNotFoundError: If tenant doesn't exist +# CacheStateError: If cached data is invalid +# """ +# +# def create_new_instance(tenant_id: int) -> ModelVariables: +# """Creator function that's called when cache miss occurs""" +# return ModelVariables(tenant_id) # This will initialize fresh variables +# +# return cache_manager.model_vars_cache.get( +# create_new_instance, # Function to create new instance if needed +# tenant_id=tenant_id # Parameters passed to both get() and create_new_instance +# ) diff --git a/common/utils/os_utils.py b/common/utils/os_utils.py index 9f1354f..06c9342 100644 --- a/common/utils/os_utils.py +++ b/common/utils/os_utils.py @@ -1,4 +1,6 @@ import os +import sys + import gevent import time from flask import current_app @@ -28,3 +30,17 @@ def sync_folder(file_path): dir_fd = os.open(file_path, os.O_RDONLY) os.fsync(dir_fd) os.close(dir_fd) + + +def get_project_root(): + """Get the root directory of the project.""" + # Use the module that's actually running (not this file) + module = sys.modules['__main__'] + if hasattr(module, '__file__'): + # Get the path to the main module + main_path = os.path.abspath(module.__file__) + # Get the root directory (where the main module is located) + return os.path.dirname(main_path) + else: + # Fallback: use current working directory + return os.getcwd() diff --git a/common/utils/security.py b/common/utils/security.py index 6aa05fa..b453421 100644 --- a/common/utils/security.py +++ b/common/utils/security.py @@ -4,7 +4,6 @@ from common.models.user import Tenant # Definition of Trigger Handlers def set_tenant_session_data(sender, user, **kwargs): - current_app.logger.debug(f"Setting tenant session data for user {user.id}") tenant = Tenant.query.filter_by(id=user.tenant_id).first() session['tenant'] = tenant.to_dict() session['default_language'] = tenant.default_language diff --git a/common/utils/security_utils.py b/common/utils/security_utils.py index f86bad1..43198a0 100644 --- a/common/utils/security_utils.py +++ b/common/utils/security_utils.py @@ -11,7 +11,7 @@ def confirm_token(token, expiration=3600): try: email = serializer.loads(token, salt=current_app.config['SECURITY_PASSWORD_SALT'], max_age=expiration) except Exception as e: - current_app.logger.debug(f'Error confirming token: {e}') + current_app.logger.error(f'Error confirming token: {e}') raise return email @@ -35,14 +35,11 @@ def generate_confirmation_token(email): def send_confirmation_email(user): - current_app.logger.debug(f'Sending confirmation email to {user.email}') - if not test_smtp_connection(): raise Exception("Failed to connect to SMTP server") token = generate_confirmation_token(user.email) confirm_url = prefixed_url_for('security_bp.confirm_email', token=token, _external=True) - current_app.logger.debug(f'Confirmation URL: {confirm_url}') html = render_template('email/activate.html', confirm_url=confirm_url) subject = "Please confirm your email" @@ -56,10 +53,8 @@ def send_confirmation_email(user): def send_reset_email(user): - current_app.logger.debug(f'Sending reset email to {user.email}') token = generate_reset_token(user.email) reset_url = prefixed_url_for('security_bp.reset_password', token=token, _external=True) - current_app.logger.debug(f'Reset URL: {reset_url}') html = render_template('email/reset_password.html', reset_url=reset_url) subject = "Reset Your Password" diff --git a/common/utils/string_list_converter.py b/common/utils/string_list_converter.py new file mode 100644 index 0000000..a71f754 --- /dev/null +++ b/common/utils/string_list_converter.py @@ -0,0 +1,112 @@ +from typing import List, Union +import re + + +class StringListConverter: + """Utility class for converting between comma-separated strings and lists""" + + @staticmethod + def string_to_list(input_string: Union[str, None], allow_empty: bool = True) -> List[str]: + """ + Convert a comma-separated string to a list of strings. + + Args: + input_string: Comma-separated string to convert + allow_empty: If True, returns empty list for None/empty input + If False, raises ValueError for None/empty input + + Returns: + List of stripped strings + + Raises: + ValueError: If input is None/empty and allow_empty is False + """ + if not input_string: + if allow_empty: + return [] + raise ValueError("Input string cannot be None or empty") + + return [item.strip() for item in input_string.split(',') if item.strip()] + + @staticmethod + def list_to_string(input_list: Union[List[str], None], allow_empty: bool = True) -> str: + """ + Convert a list of strings to a comma-separated string. + + Args: + input_list: List of strings to convert + allow_empty: If True, returns empty string for None/empty input + If False, raises ValueError for None/empty input + + Returns: + Comma-separated string + + Raises: + ValueError: If input is None/empty and allow_empty is False + """ + if not input_list: + if allow_empty: + return '' + raise ValueError("Input list cannot be None or empty") + + return ', '.join(str(item).strip() for item in input_list) + + @staticmethod + def validate_format(input_string: str, + allowed_chars: str = r'a-zA-Z0-9_\-', + min_length: int = 1, + max_length: int = 50) -> bool: + """ + Validate the format of items in a comma-separated string. + + Args: + input_string: String to validate + allowed_chars: String of allowed characters (for regex pattern) + min_length: Minimum length for each item + max_length: Maximum length for each item + + Returns: + bool: True if format is valid, False otherwise + """ + if not input_string: + return False + + # Create regex pattern for individual items + pattern = f'^[{allowed_chars}]{{{min_length},{max_length}}}$' + + try: + # Convert to list and check each item + items = StringListConverter.string_to_list(input_string) + return all(bool(re.match(pattern, item)) for item in items) + except Exception: + return False + + @staticmethod + def validate_and_convert(input_string: str, + allowed_chars: str = r'a-zA-Z0-9_\-', + min_length: int = 1, + max_length: int = 50) -> List[str]: + """ + Validate and convert a comma-separated string to a list. + + Args: + input_string: String to validate and convert + allowed_chars: String of allowed characters (for regex pattern) + min_length: Minimum length for each item + max_length: Maximum length for each item + + Returns: + List of validated and converted strings + + Raises: + ValueError: If input string format is invalid + """ + if not StringListConverter.validate_format( + input_string, allowed_chars, min_length, max_length + ): + raise ValueError( + f"Invalid format. Items must be {min_length}-{max_length} characters " + f"long and contain only these characters: {allowed_chars}" + ) + + return StringListConverter.string_to_list(input_string) \ No newline at end of file diff --git a/common/utils/view_assistants.py b/common/utils/view_assistants.py index 02c57db..8220018 100644 --- a/common/utils/view_assistants.py +++ b/common/utils/view_assistants.py @@ -44,7 +44,7 @@ def form_validation_failed(request, form): for fieldName, errorMessages in form.errors.items(): for err in errorMessages: flash(f"Error in {fieldName}: {err}", 'danger') - current_app.logger.debug(f"Error in {fieldName}: {err}") + current_app.logger.error(f"Error in {fieldName}: {err}") def form_to_dict(form): diff --git a/config/catalog_types.py b/config/catalog_types.py index 6155a60..e717e8a 100644 --- a/config/catalog_types.py +++ b/config/catalog_types.py @@ -3,7 +3,8 @@ CATALOG_TYPES = { "STANDARD_CATALOG": { "name": "Standard Catalog", "Description": "A Catalog with information in Evie's Library, to be considered as a whole", - "configuration": {} + "configuration": {}, + "document_version_configurations": [] }, "DOSSIER": { "name": "Dossier Catalog", @@ -21,31 +22,6 @@ CATALOG_TYPES = { - min_value/max_value: range limits (for numeric types only)""", "required": True, "default": {}, - "field_properties": { - "type": { - "allowed_values": ["string", "integer", "float", "date", "enum"], - "required": True - }, - "required": { - "type": "boolean", - "default": False - }, - "description": { - "type": "string" - }, - "allowed_values": { - "type": "list", - "description": "For enum type fields only" - }, - "min_value": { - "type": "number", - "description": "For numeric fields only" - }, - "max_value": { - "type": "number", - "description": "For numeric fields only" - } - } } }, "document_version_configurations": ["tagging_fields"] diff --git a/config/config.py b/config/config.py index 201baa7..0adab65 100644 --- a/config/config.py +++ b/config/config.py @@ -68,9 +68,6 @@ class Config(object): ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', } - # Load prompt templates dynamically - PROMPT_TEMPLATES = {model: load_prompt_templates(model) for model in SUPPORTED_LLMS} - # Annotation text chunk length ANNOTATION_TEXT_CHUNK_LENGTH = { 'openai.gpt-4o': 10000, @@ -87,9 +84,6 @@ class Config(object): # Anthropic API Keys ANTHROPIC_API_KEY = environ.get('ANTHROPIC_API_KEY') - # Portkey API Keys - PORTKEY_API_KEY = environ.get('PORTKEY_API_KEY') - # Celery settings CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' @@ -181,13 +175,21 @@ class DevConfig(Config): # file upload settings # UPLOAD_FOLDER = '/app/tenant_files' + # Redis Settings + REDIS_URL = 'redis' + REDIS_PORT = '6379' + REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}' + # Celery settings # eveai_app Redis Settings - CELERY_BROKER_URL = 'redis://redis:6379/0' - CELERY_RESULT_BACKEND = 'redis://redis:6379/0' + CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0' + CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0' # eveai_chat Redis Settings - CELERY_BROKER_URL_CHAT = 'redis://redis:6379/3' - CELERY_RESULT_BACKEND_CHAT = 'redis://redis:6379/3' + CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' + CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' + # eveai_chat_workers cache Redis Settings + CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4' + # Unstructured settings # UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw' @@ -195,7 +197,7 @@ class DevConfig(Config): # UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general' # SocketIO settings - SOCKETIO_MESSAGE_QUEUE = 'redis://redis:6379/1' + SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1' SOCKETIO_CORS_ALLOWED_ORIGINS = '*' SOCKETIO_LOGGER = True SOCKETIO_ENGINEIO_LOGGER = True @@ -211,7 +213,7 @@ class DevConfig(Config): GC_CRYPTO_KEY = 'envelope-encryption-key' # Session settings - SESSION_REDIS = redis.from_url('redis://redis:6379/2') + SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') # PATH settings ffmpeg_path = '/usr/bin/ffmpeg' @@ -278,6 +280,8 @@ class ProdConfig(Config): # eveai_chat Redis Settings CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' + # eveai_chat_workers cache Redis Settings + CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4' # Session settings SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') diff --git a/config/logging_config.py b/config/logging_config.py index 6415b22..e44f83e 100644 --- a/config/logging_config.py +++ b/config/logging_config.py @@ -1,4 +1,8 @@ +import json import os +from datetime import datetime as dt, timezone as tz + +from flask import current_app from graypy import GELFUDPHandler import logging import logging.config @@ -9,24 +13,173 @@ GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201)) env = os.environ.get('FLASK_ENV', 'development') -class CustomLogRecord(logging.LogRecord): +class TuningLogRecord(logging.LogRecord): + """Extended LogRecord that handles both tuning and business event logging""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # Initialize extra fields after parent initialization + self._extra_fields = {} + self._is_tuning_log = False + self._tuning_type = None + self._tuning_tenant_id = None + self._tuning_catalog_id = None + self._tuning_specialist_id = None + self._tuning_retriever_id = None + self._tuning_processor_id = None self.component = os.environ.get('COMPONENT_NAME', 'eveai_app') - def __setattr__(self, name, value): - if name not in {'event_type', 'tenant_id', 'trace_id', 'span_id', 'span_name', 'parent_span_id', - 'document_version_id', 'chat_session_id', 'interaction_id', 'environment'}: - super().__setattr__(name, value) + def getMessage(self): + """ + Override getMessage to handle both string and dict messages + """ + msg = self.msg + if self.args: + msg = msg % self.args + return msg + + @property + def is_tuning_log(self): + return self._is_tuning_log + + @is_tuning_log.setter + def is_tuning_log(self, value): + object.__setattr__(self, '_is_tuning_log', value) + + @property + def tuning_type(self): + return self._tuning_type + + @tuning_type.setter + def tuning_type(self, value): + object.__setattr__(self, '_tuning_type', value) + + def get_tuning_data(self): + """Get all tuning-related data if this is a tuning log""" + if not self._is_tuning_log: + return {} + + return { + 'is_tuning_log': self._is_tuning_log, + 'tuning_type': self._tuning_type, + 'tuning_tenant_id': self._tuning_tenant_id, + 'tuning_catalog_id': self._tuning_catalog_id, + 'tuning_specialist_id': self._tuning_specialist_id, + 'tuning_retriever_id': self._tuning_retriever_id, + 'tuning_processor_id': self._tuning_processor_id, + } + + def set_tuning_data(self, tenant_id=None, catalog_id=None, specialist_id=None, + retriever_id=None, processor_id=None): + """Set tuning-specific data""" + object.__setattr__(self, '_tuning_tenant_id', tenant_id) + object.__setattr__(self, '_tuning_catalog_id', catalog_id) + object.__setattr__(self, '_tuning_specialist_id', specialist_id) + object.__setattr__(self, '_tuning_retriever_id', retriever_id) + object.__setattr__(self, '_tuning_processor_id', processor_id) -def custom_log_record_factory(*args, **kwargs): - record = CustomLogRecord(*args, **kwargs) - return record +class TuningFormatter(logging.Formatter): + """Universal formatter for all tuning logs""" + + def __init__(self, fmt=None, datefmt=None): + super().__init__(fmt or '%(asctime)s [%(levelname)s] %(name)s: %(message)s', + datefmt or '%Y-%m-%d %H:%M:%S') + + def format(self, record): + # First format with the default formatter to handle basic fields + formatted_msg = super().format(record) + + # If this is a tuning log, add the additional context + if getattr(record, 'is_tuning_log', False): + try: + identifiers = [] + if hasattr(record, 'tenant_id') and record.tenant_id: + identifiers.append(f"Tenant: {record.tenant_id}") + if hasattr(record, 'catalog_id') and record.catalog_id: + identifiers.append(f"Catalog: {record.catalog_id}") + if hasattr(record, 'processor_id') and record.processor_id: + identifiers.append(f"Processor: {record.processor_id}") + + formatted_msg = ( + f"{formatted_msg}\n" + f"[TUNING {record.tuning_type}] [{' | '.join(identifiers)}]" + ) + + if hasattr(record, 'tuning_data') and record.tuning_data: + formatted_msg += f"\nData: {json.dumps(record.tuning_data, indent=2)}" + + except Exception as e: + return f"{formatted_msg} (Error formatting tuning data: {str(e)})" + + return formatted_msg + + +class GraylogFormatter(logging.Formatter): + """Maintains existing Graylog formatting while adding tuning fields""" + + def format(self, record): + if getattr(record, 'is_tuning_log', False): + # Add tuning-specific fields to Graylog + record.tuning_fields = { + 'is_tuning_log': True, + 'tuning_type': record.tuning_type, + 'tenant_id': record.tenant_id, + 'catalog_id': record.catalog_id, + 'specialist_id': record.specialist_id, + 'retriever_id': record.retriever_id, + 'processor_id': record.processor_id, + } + return super().format(record) + + +class TuningLogger: + """Helper class to manage tuning logs with consistent structure""" + + def __init__(self, logger_name, tenant_id=None, catalog_id=None, specialist_id=None, retriever_id=None, processor_id=None): + self.logger = logging.getLogger(logger_name) + self.tenant_id = tenant_id + self.catalog_id = catalog_id + self.specialist_id = specialist_id + self.retriever_id = retriever_id + self.processor_id = processor_id + + def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DEBUG): + """Log a tuning event with structured data""" + try: + # Create a standard LogRecord for tuning + record = logging.LogRecord( + name=self.logger.name, + level=level, + pathname='', + lineno=0, + msg=message, + args=(), + exc_info=None + ) + + # Add tuning-specific attributes + record.is_tuning_log = True + record.tuning_type = tuning_type + record.tenant_id = self.tenant_id + record.catalog_id = self.catalog_id + record.specialist_id = self.specialist_id + record.retriever_id = self.retriever_id + record.processor_id = self.processor_id + + if data: + record.tuning_data = data + + # Process the record + self.logger.handle(record) + + except Exception as e: + fallback_logger = logging.getLogger('eveai_workers') + fallback_logger.exception(f"Failed to log tuning message: {str(e)}") # Set the custom log record factory -logging.setLogRecordFactory(custom_log_record_factory) +logging.setLogRecordFactory(TuningLogRecord) LOGGING = { @@ -38,7 +191,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_app.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_workers': { @@ -46,7 +199,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_workers.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_chat': { @@ -54,7 +207,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_chat.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_chat_workers': { @@ -62,7 +215,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_chat_workers.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_api': { @@ -70,7 +223,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_api.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_beat': { @@ -78,7 +231,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_beat.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_entitlements': { @@ -86,7 +239,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_entitlements.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_sqlalchemy': { @@ -94,7 +247,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/sqlalchemy.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_mailman': { @@ -102,7 +255,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/mailman.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_security': { @@ -110,7 +263,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/security.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_rag_tuning': { @@ -118,7 +271,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/rag_tuning.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_embed_tuning': { @@ -126,7 +279,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/embed_tuning.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'file_business_events': { @@ -134,7 +287,7 @@ LOGGING = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/business_events.log', 'maxBytes': 1024 * 1024 * 1, # 1MB - 'backupCount': 10, + 'backupCount': 2, 'formatter': 'standard', }, 'console': { @@ -142,25 +295,38 @@ LOGGING = { 'level': 'DEBUG', 'formatter': 'standard', }, + 'tuning_file': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': 'logs/tuning.log', + 'maxBytes': 1024 * 1024 * 3, # 3MB + 'backupCount': 3, + 'formatter': 'tuning', + }, 'graylog': { 'level': 'DEBUG', 'class': 'graypy.GELFUDPHandler', 'host': GRAYLOG_HOST, 'port': GRAYLOG_PORT, - 'debugging_fields': True, # Set to True if you want to include debugging fields - 'extra_fields': True, # Set to True if you want to include extra fields + 'debugging_fields': True, + 'formatter': 'graylog' }, }, 'formatters': { 'standard': { - 'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' - '[Thread: %(threadName)s]: %(message)s' + 'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d]: %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' }, 'graylog': { 'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' '[Thread: %(threadName)s]: %(message)s', 'datefmt': '%Y-%m-%d %H:%M:%S', + '()': GraylogFormatter }, + 'tuning': { + '()': TuningFormatter, + 'datefmt': '%Y-%m-%d %H:%M:%S UTC' + } }, 'loggers': { 'eveai_app': { # logger for the eveai_app @@ -213,21 +379,17 @@ LOGGING = { 'level': 'DEBUG', 'propagate': False }, - 'rag_tuning': { # logger for the rag_tuning - 'handlers': ['file_rag_tuning', 'graylog', ] if env == 'production' else ['file_rag_tuning', ], - 'level': 'DEBUG', - 'propagate': False - }, - 'embed_tuning': { # logger for the embed_tuning - 'handlers': ['file_embed_tuning', 'graylog', ] if env == 'production' else ['file_embed_tuning', ], - 'level': 'DEBUG', - 'propagate': False - }, 'business_events': { 'handlers': ['file_business_events', 'graylog'], 'level': 'DEBUG', 'propagate': False }, + # Single tuning logger + 'tuning': { + 'handlers': ['tuning_file', 'graylog'] if env == 'production' else ['tuning_file'], + 'level': 'DEBUG', + 'propagate': False, + }, '': { # root logger 'handlers': ['console'], 'level': 'WARNING', # Set higher level for root to minimize noise diff --git a/config/processor_types.py b/config/processor_types.py new file mode 100644 index 0000000..b0ec047 --- /dev/null +++ b/config/processor_types.py @@ -0,0 +1,56 @@ +# Catalog Types +PROCESSOR_TYPES = { + "HTML_PROCESSOR": { + "name": "HTML Processor", + "file_types": "html", + "Description": "A processor for HTML files", + "configuration": { + "html_tags": { + "name": "HTML Tags", + "type": "string", + "description": "A comma-separated list of HTML tags", + "required": True, + "default": "p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td" + }, + "html_end_tags": { + "name": "HTML End Tags", + "type": "string", + "description": "A comma-separated list of HTML end tags (where can the chunk end)", + "required": True, + "default": "p, li, table" + }, + "html_included_elements": { + "name": "HTML Included Elements", + "type": "string", + "description": "A comma-separated list of elements to be included", + "required": True, + "default": "article, main" + }, + "html_excluded_elements": { + "name": "HTML Excluded Elements", + "type": "string", + "description": "A comma-separated list of elements to be excluded", + "required": False, + "default": "header, footer, nav, script" + }, + "html_excluded_classes": { + "name": "HTML Excluded Classes", + "type": "string", + "description": "A comma-separated list of classes to be excluded", + "required": False, + }, + }, + }, + "PDF_PROCESSOR": { + "name": "PDF Processor", + "file_types": "pdf", + "Description": "A Processor for PDF files", + "configuration": {} + }, + "AUDIO_PROCESSOR": { + "name": "AUDIO Processor", + "file_types": "mp3, mp4, ogg", + "Description": "A Processor for audio files", + "configuration": {} + }, +} diff --git a/config/prompts/anthropic/claude-3-5-sonnet.yaml b/config/prompts/anthropic/claude-3-5-sonnet.yaml deleted file mode 100644 index c2f2106..0000000 --- a/config/prompts/anthropic/claude-3-5-sonnet.yaml +++ /dev/null @@ -1,88 +0,0 @@ -html_parse: | - You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. - - # Best practices are: - - Respect wordings and language(s) used in the HTML. - - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. - - Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list. - - Be careful of encoding of the text. Everything needs to be human readable. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text. - - HTML is between triple backticks. - - ```{html}``` - -pdf_parse: | - You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. - - # Best practices are: - - Respect wordings and language(s) used in the PDF. - - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. - - When headings are numbered, show the numbering and define the header level. - - A new item is started when a is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily). - - Paragraphs are to be stripped of newlines so they become easily readable. - - Be careful of encoding of the text. Everything needs to be human readable. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text. - - PDF content is between triple backticks. - - ```{pdf_content}``` - -summary: | - Write a concise summary of the text in {language}. The text is delimited between triple backticks. - ```{text}``` - -rag: | - Answer the question based on the following context, delimited between triple backticks. - {tenant_context} - Use the following {language} in your communication, and cite the sources used. - If the question cannot be answered using the given context, say "I have insufficient information to answer this question." - Context: - ```{context}``` - Question: - {question} - -history: | - You are a helpful assistant that details a question based on a previous context, - in such a way that the question is understandable without the previous context. - The context is a conversation history, with the HUMAN asking questions, the AI answering questions. - The history is delimited between triple backticks. - You answer by stating the question in {language}. - History: - ```{history}``` - Question to be detailed: - {question} - -encyclopedia: | - You have a lot of background knowledge, and as such you are some kind of - 'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question. - If not, say you do not have sufficient information to answer the question. Use the {language} in your communication. - Question: - {question} - -transcript: | - """You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. Your task is to process and improve the given transcript, not to summarize it. - - IMPORTANT INSTRUCTIONS: - 1. DO NOT summarize the transcript and don't make your own interpretations. Return the FULL, COMPLETE transcript with improvements. - 2. Improve any errors in the transcript based on context. - 3. Respect the original wording and language(s) used in the transcription. Main Language used is {language}. - 4. Divide the transcript into paragraphs for better readability. Each paragraph ONLY contains ORIGINAL TEXT. - 5. Group related paragraphs into logical sections. - 6. Add appropriate headers (using markdown syntax) to each section in {language}. - 7. We do not need an overall title. Just add logical headers - 8. Ensure that the entire transcript is included in your response, from start to finish. - - REMEMBER: - - Your output should be the complete transcript in markdown format, NOT A SUMMARY OR ANALYSIS. - - Include EVERYTHING from the original transcript, just organized and formatted better. - - Just return the markdown version of the transcript, without any other text such as an introduction or a summary. - - Here is the transcript to process (between triple backticks): - - ```{transcript}``` - - Process this transcript according to the instructions above and return the full, formatted markdown version. - """ \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o-mini.yaml b/config/prompts/openai/gpt-4o-mini.yaml deleted file mode 100644 index 2413299..0000000 --- a/config/prompts/openai/gpt-4o-mini.yaml +++ /dev/null @@ -1,79 +0,0 @@ -html_parse: | - You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. - - # Best practices are: - - Respect wordings and language(s) used in the HTML. - - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. - - Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list. - - Be careful of encoding of the text. Everything needs to be human readable. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text. - - HTML is between triple backquotes. - - ```{html}``` - -pdf_parse: | - You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. - - # Best practices are: - - Respect wordings and language(s) used in the PDF. - - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. - - When headings are numbered, show the numbering and define the header level. - - A new item is started when a is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily). - - Paragraphs are to be stripped of newlines so they become easily readable. - - Be careful of encoding of the text. Everything needs to be human readable. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text. - - PDF content is between triple backquotes. - - ```{pdf_content}``` - -summary: | - Write a concise summary of the text in {language}. The text is delimited between triple backquotes. - ```{text}``` - -rag: | - Answer the question based on the following context, delimited between triple backquotes. - {tenant_context} - Use the following {language} in your communication, and cite the sources used. - If the question cannot be answered using the given context, say "I have insufficient information to answer this question." - Context: - ```{context}``` - Question: - {question} - -history: | - You are a helpful assistant that details a question based on a previous context, - in such a way that the question is understandable without the previous context. - The context is a conversation history, with the HUMAN asking questions, the AI answering questions. - The history is delimited between triple backquotes. - You answer by stating the question in {language}. - History: - ```{history}``` - Question to be detailed: - {question} - -encyclopedia: | - You have a lot of background knowledge, and as such you are some kind of - 'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question. - If not, say you do not have sufficient information to answer the question. Use the {language} in your communication. - Question: - {question} - -transcript: | - You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material. - - # Best practices and steps are: - - Respect wordings and language(s) used in the transcription. Main language is {language}. - - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking. - - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. - - annotate the text to identify these logical parts using headings in {language}. - - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text. - - The transcript is between triple backquotes. - - ```{transcript}``` \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o.yaml b/config/prompts/openai/gpt-4o.yaml deleted file mode 100644 index ee1d513..0000000 --- a/config/prompts/openai/gpt-4o.yaml +++ /dev/null @@ -1,84 +0,0 @@ -html_parse: | - You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. - - # Best practices are: - - Respect wordings and language(s) used in the HTML. - - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. - - Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list. - - Be careful of encoding of the text. Everything needs to be human readable. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text. - - HTML is between triple backquotes. - - ```{html}``` - -pdf_parse: | - You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. - The content you get is already processed (some markdown already generated), but needs to be corrected. For large files, you may receive only portions of the full file. Consider this when processing the content. - - # Best practices are: - - Respect wordings and language(s) used in the provided content. - - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. - - When headings are numbered, show the numbering and define the header level. You may have to correct current header levels, as preprocessing is known to make errors. - - A new item is started when a is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily). - - Paragraphs are to be stripped of newlines so they become easily readable. - - Be careful of encoding of the text. Everything needs to be human readable. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text. - - PDF content is between triple backquotes. - - ```{pdf_content}``` - -summary: | - Write a concise summary of the text in {language}. The text is delimited between triple backquotes. - ```{text}``` - -rag: | - Answer the question based on the following context, delimited between triple backquotes. - {tenant_context} - Use the following {language} in your communication, and cite the sources used. - If the question cannot be answered using the given context, say "I have insufficient information to answer this question." - Context: - ```{context}``` - Question: - {question} - -history: | - You are a helpful assistant that details a question based on a previous context, - in such a way that the question is understandable without the previous context. - The context is a conversation history, with the HUMAN asking questions, the AI answering questions. - The history is delimited between triple backquotes. - You answer by stating the question in {language}. - History: - ```{history}``` - Question to be detailed: - {question} - -encyclopedia: | - You have a lot of background knowledge, and as such you are some kind of - 'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question. - If not, say you do not have sufficient information to answer the question. Use the {language} in your communication. - Question: - {question} - -transcript: | - You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material. - You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk. - - - # Best practices and steps are: - - Respect wordings and language(s) used in the transcription. Main language is {language}. - - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking. - - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences. - - annotate the text to identify these logical parts using headings in {language}. - - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription. - - Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text. - - The transcript is between triple backquotes. - - $$${previous_part}$$$ - - ```{transcript}``` \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/encyclopedia/1.0.0.yaml b/config/prompts/openai/gpt-4o/encyclopedia/1.0.0.yaml new file mode 100644 index 0000000..c362492 --- /dev/null +++ b/config/prompts/openai/gpt-4o/encyclopedia/1.0.0.yaml @@ -0,0 +1,12 @@ +version: "1.0.0" +content: | + You have a lot of background knowledge, and as such you are some kind of + 'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question. + If not, say you do not have sufficient information to answer the question. Use the {language} in your communication. + Question: + {question} +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "A background information retriever for Evie" + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/history/1.0.0.yaml b/config/prompts/openai/gpt-4o/history/1.0.0.yaml new file mode 100644 index 0000000..06862fc --- /dev/null +++ b/config/prompts/openai/gpt-4o/history/1.0.0.yaml @@ -0,0 +1,16 @@ +version: "1.0.0" +content: | + You are a helpful assistant that details a question based on a previous context, + in such a way that the question is understandable without the previous context. + The context is a conversation history, with the HUMAN asking questions, the AI answering questions. + The history is delimited between triple backquotes. + You answer by stating the question in {language}. + History: + ```{history}``` + Question to be detailed: + {question} +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "Prompt to further detail a question based on the previous conversation" + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/html_parse/1.0.0.yaml b/config/prompts/openai/gpt-4o/html_parse/1.0.0.yaml new file mode 100644 index 0000000..f1bb745 --- /dev/null +++ b/config/prompts/openai/gpt-4o/html_parse/1.0.0.yaml @@ -0,0 +1,20 @@ +version: "1.0.0" +content: | + You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. + + # Best practices are: + - Respect wordings and language(s) used in the HTML. + - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. + - Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list. + - Be careful of encoding of the text. Everything needs to be human readable. + + Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text. + + HTML is between triple backquotes. + + ```{html}``` +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "An aid in transforming HTML-based inputs to markdown" + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/pdf_parse/1.0.0.yaml b/config/prompts/openai/gpt-4o/pdf_parse/1.0.0.yaml new file mode 100644 index 0000000..6af358a --- /dev/null +++ b/config/prompts/openai/gpt-4o/pdf_parse/1.0.0.yaml @@ -0,0 +1,23 @@ +version: "1.0.0" +content: | + You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. + The content you get is already processed (some markdown already generated), but needs to be corrected. For large files, you may receive only portions of the full file. Consider this when processing the content. + + # Best practices are: + - Respect wordings and language(s) used in the provided content. + - The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected. + - When headings are numbered, show the numbering and define the header level. You may have to correct current header levels, as preprocessing is known to make errors. + - A new item is started when a is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily). + - Paragraphs are to be stripped of newlines so they become easily readable. + - Be careful of encoding of the text. Everything needs to be human readable. + + Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text. + + PDF content is between triple backquotes. + + ```{pdf_content}``` +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "A assistant to parse PDF-content into markdown" + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/rag/1.0.0.yaml b/config/prompts/openai/gpt-4o/rag/1.0.0.yaml new file mode 100644 index 0000000..24ddbbc --- /dev/null +++ b/config/prompts/openai/gpt-4o/rag/1.0.0.yaml @@ -0,0 +1,15 @@ +version: "1.0.0" +content: | + Answer the question based on the following context, delimited between triple backquotes. + {tenant_context} + Use the following {language} in your communication, and cite the sources used at the end of the full conversation. + If the question cannot be answered using the given context, say "I have insufficient information to answer this question." + Context: + ```{context}``` + Question: + {question} +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "The Main RAG retriever" + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/summary/1.0.0.yaml b/config/prompts/openai/gpt-4o/summary/1.0.0.yaml new file mode 100644 index 0000000..c1e5b30 --- /dev/null +++ b/config/prompts/openai/gpt-4o/summary/1.0.0.yaml @@ -0,0 +1,9 @@ +version: "1.0.0" +content: | + Write a concise summary of the text in {language}. The text is delimited between triple backquotes. + ```{text}``` +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "An assistant to create a summary when multiple chunks are required for 1 file" + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/prompts/openai/gpt-4o/transcript/1.0.0.yaml b/config/prompts/openai/gpt-4o/transcript/1.0.0.yaml new file mode 100644 index 0000000..3195ba4 --- /dev/null +++ b/config/prompts/openai/gpt-4o/transcript/1.0.0.yaml @@ -0,0 +1,25 @@ +version: "1.0.0" +content: | + You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material. + You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk. + + + # Best practices and steps are: + - Respect wordings and language(s) used in the transcription. Main language is {language}. + - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking. + - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences. + - annotate the text to identify these logical parts using headings in {language}. + - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription. + + Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text. + + The transcript is between triple backquotes. + + $$${previous_part}$$$ + + ```{transcript}``` +metadata: + author: "Josako" + date_added: "2024-11-10" + description: "An assistant to transform a transcript to markdown." + changes: "Initial version migrated from flat file structure" \ No newline at end of file diff --git a/config/retriever_types.py b/config/retriever_types.py index 8a743c2..14df534 100644 --- a/config/retriever_types.py +++ b/config/retriever_types.py @@ -18,6 +18,14 @@ RETRIEVER_TYPES = { "required": True, "default": 0.3, }, + }, + "arguments": { + "query": { + "name": "query", + "type": "str", + "description": "Query to retrieve embeddings", + "required": True, + }, } } } diff --git a/config/specialist_types.py b/config/specialist_types.py index 14b1d7c..d0c6d02 100644 --- a/config/specialist_types.py +++ b/config/specialist_types.py @@ -10,6 +10,13 @@ SPECIALIST_TYPES = { "description": "The context to be used by the specialist.", "required": False, }, + "temperature": { + "name": "Temperature", + "type": "number", + "description": "The inference temperature to be used by the specialist.", + "required": False, + "default": 0.3 + } }, "arguments": { "language": { @@ -18,6 +25,38 @@ SPECIALIST_TYPES = { "description": "Language code to be used for receiving questions and giving answers", "required": True, }, + "query": { + "name": "query", + "type": "str", + "description": "Query to answer", + "required": True, + } + }, + "results": { + "detailed_query": { + "name": "detailed_query", + "type": "str", + "description": "The query detailed with the Chat Session History.", + "required": True, + }, + "answer": { + "name": "answer", + "type": "str", + "description": "Answer to the query", + "required": True, + }, + "citations": { + "name": "citations", + "type": "List[str]", + "description": "List of citations", + "required": False, + }, + "insufficient_info": { + "name": "insufficient_info", + "type": "bool", + "description": "Whether or not the query is insufficient info", + "required": True, + }, } } -} +} \ No newline at end of file diff --git a/docker/compose_dev.yaml b/docker/compose_dev.yaml index 18fc5c8..6b88f1c 100644 --- a/docker/compose_dev.yaml +++ b/docker/compose_dev.yaml @@ -27,7 +27,6 @@ x-common-variables: &common-variables OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7' GROQ_API_KEY: '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' ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2' - PORTKEY_API_KEY: 'T2Dt4QTpgCvWxa1OftYCJtj7NcDZ' JWT_SECRET_KEY: 'bsdMkmQ8ObfMD52yAFg4trrvjgjMhuIqg2fjDpD/JqvgY0ccCcmlsEnVFmR79WPiLKEA3i8a5zmejwLZKl4v9Q==' API_ENCRYPTION_KEY: 'xfF5369IsredSrlrYZqkM9ZNrfUASYYS6TCcAR9UKj4=' MINIO_ENDPOINT: minio:9000 @@ -90,7 +89,7 @@ services: - ../migrations:/app/migrations - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -124,7 +123,7 @@ services: - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -154,7 +153,7 @@ services: - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -186,7 +185,7 @@ services: - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -214,7 +213,7 @@ services: - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -248,7 +247,7 @@ services: - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: redis: condition: service_healthy @@ -272,7 +271,7 @@ services: - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - eveai_logs:/app/logs + - ./eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -308,8 +307,8 @@ services: redis: image: redis:7.2.5 restart: always - expose: - - 6379 + ports: + - "6379:6379" volumes: - ./db/redis:/data healthcheck: diff --git a/docker/compose_stackhero.yaml b/docker/compose_stackhero.yaml index d1c47da..095c302 100644 --- a/docker/compose_stackhero.yaml +++ b/docker/compose_stackhero.yaml @@ -31,7 +31,6 @@ x-common-variables: &common-variables OPENAI_API_KEY: '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' GROQ_API_KEY: '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' ANTHROPIC_API_KEY: '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' - PORTKEY_API_KEY: 'XvmvBFIVbm76opUxA7MNP14QmdQj' JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408' API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0=' GRAYLOG_HOST: de4zvu.stackhero-network.com diff --git a/eveai_api/__init__.py b/eveai_api/__init__.py index 3b4a0b0..2d6cb53 100644 --- a/eveai_api/__init__.py +++ b/eveai_api/__init__.py @@ -52,37 +52,16 @@ def create_app(config_file=None): @app.before_request def before_request(): - app.logger.debug(f'Before request: {request.method} {request.path}') - app.logger.debug(f'Request URL: {request.url}') - app.logger.debug(f'Request headers: {dict(request.headers)}') - - # Log JSON data if the content type is application/json - if request.is_json: - app.logger.debug(f'JSON data: {request.json}') - - # Log raw data for other content types - if request.data: - app.logger.debug(f'Raw data: {request.data}') - - # Check if this is a request to the token endpoint - if request.path == '/api/v1/auth/token' and request.method == 'POST': - app.logger.debug('Token request detected, skipping JWT verification') - return - # Check if this a health check request if request.path.startswith('/_healthz') or request.path.startswith('/healthz'): - app.logger.debug('Health check request detected, skipping JWT verification') + pass else: try: verify_jwt_in_request(optional=True) tenant_id = get_jwt_identity() - app.logger.debug(f'Tenant ID from JWT: {tenant_id}') if tenant_id: Database(tenant_id).switch_schema() - app.logger.debug(f'Switched to schema for tenant {tenant_id}') - else: - app.logger.debug('No tenant ID found in JWT') except Exception as e: app.logger.error(f'Error in before_request: {str(e)}') # Don't raise the exception here, let the request continue diff --git a/eveai_api/api/auth.py b/eveai_api/api/auth.py index 7807038..21c3a90 100644 --- a/eveai_api/api/auth.py +++ b/eveai_api/api/auth.py @@ -30,8 +30,6 @@ class Token(Resource): """ Get JWT token """ - current_app.logger.debug(f"Token endpoint called with data: {request.json}") - try: tenant_id = auth_ns.payload['tenant_id'] api_key = auth_ns.payload['api_key'] @@ -39,17 +37,13 @@ class Token(Resource): current_app.logger.error(f"Missing required field: {e}") return {'message': f"Missing required field: {e}"}, 400 - current_app.logger.debug(f"Querying database for tenant: {tenant_id}") tenant = Tenant.query.get(tenant_id) if not tenant: current_app.logger.error(f"Tenant not found: {tenant_id}") return {'message': "Tenant not found"}, 404 - current_app.logger.debug(f"Tenant found: {tenant.id}") - try: - current_app.logger.debug("Attempting to decrypt API key") decrypted_api_key = simple_encryption.decrypt_api_key(tenant.encrypted_api_key) except Exception as e: current_app.logger.error(f"Error decrypting API key: {e}") @@ -63,9 +57,7 @@ class Token(Resource): expires_delta = current_app.config.get('JWT_ACCESS_TOKEN_EXPIRES', timedelta(minutes=15)) try: - current_app.logger.debug(f"Creating access token for tenant: {tenant_id}") access_token = create_access_token(identity=tenant_id, expires_delta=expires_delta) - current_app.logger.debug("Access token created successfully") return { 'access_token': access_token, 'expires_in': expires_delta.total_seconds() diff --git a/eveai_app/__init__.py b/eveai_app/__init__.py index 3783538..06e9c05 100644 --- a/eveai_app/__init__.py +++ b/eveai_app/__init__.py @@ -7,7 +7,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix import logging.config from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session, - minio_client, simple_encryption, metrics) + minio_client, simple_encryption, metrics, cache_manager) from common.models.user import User, Role, Tenant, TenantDomain import common.models.interaction import common.models.entitlements @@ -119,6 +119,7 @@ def register_extensions(app): simple_encryption.init_app(app) session.init_app(app) minio_client.init_app(app) + cache_manager.init_app(app) metrics.init_app(app) diff --git a/eveai_app/templates/document/edit_processor.html b/eveai_app/templates/document/edit_processor.html new file mode 100644 index 0000000..4118994 --- /dev/null +++ b/eveai_app/templates/document/edit_processor.html @@ -0,0 +1,33 @@ +{% extends 'base.html' %} +{% from "macros.html" import render_field %} + +{% block title %}Edit Processor{% endblock %} + +{% block content_title %}Edit Processor{% endblock %} +{% block content_description %}Edit a Processor (for a Catalog){% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = ['type'] %} + {% set exclude_fields = [] %} + + {% for field in form.get_static_fields() %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + + {% for collection_name, fields in form.get_dynamic_fields().items() %} + {% if fields|length > 0 %} +

{{ collection_name }}

+ {% endif %} + {% for field in fields %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + {% endfor %} + +
+{% endblock %} + +{% block content_footer %} + +{% endblock %} diff --git a/eveai_app/templates/document/processor.html b/eveai_app/templates/document/processor.html new file mode 100644 index 0000000..bc57f5c --- /dev/null +++ b/eveai_app/templates/document/processor.html @@ -0,0 +1,23 @@ +{% extends 'base.html' %} +{% from "macros.html" import render_field %} + +{% block title %}Processor Registration{% endblock %} + +{% block content_title %}Register Processor{% endblock %} +{% block content_description %}Define a new processor (for a catalog){% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = [] %} + {% set exclude_fields = [] %} + {% for field in form %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + +
+{% endblock %} + +{% block content_footer %} + +{% endblock %} diff --git a/eveai_app/templates/document/processors.html b/eveai_app/templates/document/processors.html new file mode 100644 index 0000000..38b7598 --- /dev/null +++ b/eveai_app/templates/document/processors.html @@ -0,0 +1,23 @@ +{% extends 'base.html' %} +{% from 'macros.html' import render_selectable_table, render_pagination %} + +{% block title %}Processors{% endblock %} + +{% block content_title %}Processors{% endblock %} +{% block content_description %}View Processors for Tenant{% endblock %} +{% block content_class %}
{% endblock %} + +{% block content %} +
+
+ {{ render_selectable_table(headers=["Processor ID", "Name", "Type", "Catalog ID"], rows=rows, selectable=True, id="retrieversTable") }} +
+ +
+
+
+{% endblock %} + +{% block content_footer %} + {{ render_pagination(pagination, 'document_bp.processors') }} +{% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/interaction/edit_specialist.html b/eveai_app/templates/interaction/edit_specialist.html index 807636e..be80e6a 100644 --- a/eveai_app/templates/interaction/edit_specialist.html +++ b/eveai_app/templates/interaction/edit_specialist.html @@ -24,7 +24,7 @@ {{ render_field(field, disabled_fields, exclude_fields) }} {% endfor %} {% endfor %} - + {% endblock %} diff --git a/eveai_app/templates/navbar.html b/eveai_app/templates/navbar.html index 2d48b48..548e793 100644 --- a/eveai_app/templates/navbar.html +++ b/eveai_app/templates/navbar.html @@ -83,6 +83,8 @@ {{ dropdown('Document Mgmt', 'note_stack', [ {'name': 'Add Catalog', 'url': '/document/catalog', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'All Catalogs', 'url': '/document/catalogs', 'roles': ['Super User', 'Tenant Admin']}, + {'name': 'Add Processor', 'url': '/document/processor', 'roles': ['Super User', 'Tenant Admin']}, + {'name': 'All Processors', 'url': '/document/processors', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'Add Retriever', 'url': '/document/retriever', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'All Retrievers', 'url': '/document/retrievers', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Tenant Admin']}, diff --git a/eveai_app/views/basic_views.py b/eveai_app/views/basic_views.py index ca15454..a4e9063 100644 --- a/eveai_app/views/basic_views.py +++ b/eveai_app/views/basic_views.py @@ -9,12 +9,11 @@ basic_bp = Blueprint('basic_bp', __name__) @basic_bp.before_request def log_before_request(): - current_app.logger.debug(f"Before request (basic_bp): {request.method} {request.url}") + pass @basic_bp.after_request def log_after_request(response): - current_app.logger.debug(f"After request (basic_bp): {request.method} {request.url} - Status: {response.status}") return response diff --git a/eveai_app/views/document_forms.py b/eveai_app/views/document_forms.py index 344ac13..047c621 100644 --- a/eveai_app/views/document_forms.py +++ b/eveai_app/views/document_forms.py @@ -11,6 +11,7 @@ from wtforms_sqlalchemy.fields import QuerySelectField from common.models.document import Catalog from config.catalog_types import CATALOG_TYPES +from config.processor_types import PROCESSOR_TYPES from config.retriever_types import RETRIEVER_TYPES from .dynamic_form_base import DynamicFormBase @@ -38,28 +39,14 @@ class CatalogForm(FlaskForm): # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config) type = SelectField('Catalog Type', validators=[DataRequired()]) - # Metadata fields - user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) - system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) - - # HTML Embedding Variables - html_tags = StringField('HTML Tags', validators=[DataRequired()], - default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td') - html_end_tags = StringField('HTML End Tags', validators=[DataRequired()], - default='p, li') - html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main') - html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()], - default='header, footer, nav, script') - html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()]) min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()], default=2000) max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()], default=3000) - # Chat Variables - chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)]) - chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)]) - # Tuning variables - embed_tuning = BooleanField('Enable Embedding Tuning', default=False) + + # Metadata fields + user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) + system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -74,28 +61,75 @@ class EditCatalogForm(DynamicFormBase): # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config) type = StringField('Catalog Type', validators=[DataRequired()], render_kw={'readonly': True}) - # Metadata fields - user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) - system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],) - - # HTML Embedding Variables - html_tags = StringField('HTML Tags', validators=[DataRequired()], - default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td') - html_end_tags = StringField('HTML End Tags', validators=[DataRequired()], - default='p, li') - html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main') - html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()], - default='header, footer, nav, script') - html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()]) min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()], default=2000) max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()], default=3000) - # Chat Variables - chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)]) - chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)]) - # Tuning variables - embed_tuning = BooleanField('Enable Embedding Tuning', default=False) + + # Metadata fields + user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) + system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],) + + +class ProcessorForm(FlaskForm): + name = StringField('Name', validators=[DataRequired(), Length(max=50)]) + description = TextAreaField('Description', validators=[Optional()]) + + # Catalog for the Retriever + catalog = QuerySelectField( + 'Catalog ID', + query_factory=lambda: Catalog.query.all(), + allow_blank=True, + get_label='name', + validators=[Optional()], + ) + + # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config) + type = SelectField('Processor Type', validators=[DataRequired()]) + + sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)]) + + min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()], + default=2000) + max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()], + default=3000) + tuning = BooleanField('Enable Embedding Tuning', default=False) + + # Metadata fields + user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) + system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Dynamically populate the 'type' field using the constructor + self.type.choices = [(key, value['name']) for key, value in PROCESSOR_TYPES.items()] + + +class EditProcessorForm(DynamicFormBase): + name = StringField('Name', validators=[DataRequired(), Length(max=50)]) + description = TextAreaField('Description', validators=[Optional()]) + + # Catalog for the Retriever + catalog = QuerySelectField( + 'Catalog ID', + query_factory=lambda: Catalog.query.all(), + allow_blank=True, + get_label='name', + validators=[Optional()], + ) + type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True}) + + sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)]) + + min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()], + default=2000) + max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()], + default=3000) + tuning = BooleanField('Enable Embedding Tuning', default=False) + + # Metadata fields + user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) + system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) class RetrieverForm(FlaskForm): @@ -135,22 +169,17 @@ class EditRetrieverForm(DynamicFormBase): validators=[Optional()], ) # Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config) - type = SelectField('Retriever Type', validators=[DataRequired()], render_kw={'readonly': True}) + type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True}) tuning = BooleanField('Enable Tuning', default=False) # Metadata fields user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Set the retriever type choices (loaded from config) - self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()] - class AddDocumentForm(DynamicFormBase): file = FileField('File', validators=[FileRequired(), allowed_file]) + sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)]) name = StringField('Name', validators=[Length(max=100)]) language = SelectField('Language', choices=[], validators=[Optional()]) user_context = TextAreaField('User Context', validators=[Optional()]) @@ -167,6 +196,7 @@ class AddDocumentForm(DynamicFormBase): class AddURLForm(DynamicFormBase): url = URLField('URL', validators=[DataRequired(), URL()]) + sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)]) name = StringField('Name', validators=[Length(max=100)]) language = SelectField('Language', choices=[], validators=[Optional()]) user_context = TextAreaField('User Context', validators=[Optional()]) @@ -207,6 +237,7 @@ class EditDocumentForm(FlaskForm): class EditDocumentVersionForm(DynamicFormBase): + sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)]) language = StringField('Language') user_context = TextAreaField('User Context', validators=[Optional()]) system_context = TextAreaField('System Context', validators=[Optional()]) diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index 6323381..581a151 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -14,15 +14,16 @@ from urllib.parse import urlparse, unquote import io import json -from common.models.document import Document, DocumentVersion, Catalog, Retriever +from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor from common.extensions import db, minio_client from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \ process_multiple_urls, get_documents_list, edit_document, \ edit_document_version, refresh_document from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \ EveAIDoubleURLException +from config.processor_types import PROCESSOR_TYPES from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \ - CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm + CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm from common.utils.middleware import mw_before_request from common.utils.celery_utils import current_celery from common.utils.nginx_utils import prefixed_url_for @@ -37,13 +38,11 @@ document_bp = Blueprint('document_bp', __name__, url_prefix='/document') @document_bp.before_request def log_before_request(): - current_app.logger.debug(f"Before request (document_bp): {request.method} {request.url}") + pass @document_bp.after_request def log_after_request(response): - current_app.logger.debug( - f"After request (document_bp): {request.method} {request.url} - Status: {response.status}") return response @@ -53,8 +52,6 @@ def before_request(): mw_before_request() except Exception as e: current_app.logger.error(f'Error switching schema in Document Blueprint: {e}') - for role in current_user.roles: - current_app.logger.debug(f'User {current_user.email} has role {role.name}') raise @@ -67,16 +64,6 @@ def catalog(): tenant_id = session.get('tenant').get('id') new_catalog = Catalog() form.populate_obj(new_catalog) - # Handle Embedding Variables - new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else [] - new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \ - if form.html_end_tags.data else [] - new_catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \ - if form.html_included_elements.data else [] - new_catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \ - if form.html_excluded_elements.data else [] - new_catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \ - if form.html_excluded_classes.data else [] set_logging_information(new_catalog, dt.now(tz.utc)) try: @@ -84,6 +71,8 @@ def catalog(): db.session.commit() flash('Catalog successfully added!', 'success') current_app.logger.info(f'Catalog {new_catalog.name} successfully added for tenant {tenant_id}!') + # Enable step 2 of creation of catalog - add configuration of the catalog (dependent on type) + return redirect(prefixed_url_for('document_bp.catalog', catalog_id=new_catalog.id)) except SQLAlchemyError as e: db.session.rollback() flash(f'Failed to add catalog. Error: {e}', 'danger') @@ -140,27 +129,8 @@ def edit_catalog(catalog_id): configuration_config = CATALOG_TYPES[catalog.type]["configuration"] form.add_dynamic_fields("configuration", configuration_config, catalog.configuration) - # Convert arrays to comma-separated strings for display - if request.method == 'GET': - form.html_tags.data = ', '.join(catalog.html_tags or '') - form.html_end_tags.data = ', '.join(catalog.html_end_tags or '') - form.html_included_elements.data = ', '.join(catalog.html_included_elements or '') - form.html_excluded_elements.data = ', '.join(catalog.html_excluded_elements or '') - form.html_excluded_classes.data = ', '.join(catalog.html_excluded_classes or '') - if request.method == 'POST' and form.validate_on_submit(): form.populate_obj(catalog) - # Handle Embedding Variables - catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else [] - catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \ - if form.html_end_tags.data else [] - catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \ - if form.html_included_elements.data else [] - catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \ - if form.html_excluded_elements.data else [] - catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \ - if form.html_excluded_classes.data else [] - catalog.configuration = form.get_dynamic_data('configuration') update_logging_information(catalog, dt.now(tz.utc)) try: @@ -180,6 +150,116 @@ def edit_catalog(catalog_id): return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id) +@document_bp.route('/processor', methods=['GET', 'POST']) +@roles_accepted('Super User', 'Tenant Admin') +def processor(): + form = ProcessorForm() + + if form.validate_on_submit(): + tenant_id = session.get('tenant').get('id') + new_processor = Processor() + form.populate_obj(new_processor) + new_processor.catalog_id = form.catalog.data.id + + set_logging_information(new_processor, dt.now(tz.utc)) + + try: + db.session.add(new_processor) + db.session.commit() + flash('Processor successfully added!', 'success') + current_app.logger.info(f'Processor {new_processor.name} successfully added for tenant {tenant_id}!') + # Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type) + return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=new_processor.id)) + except SQLAlchemyError as e: + db.session.rollback() + flash(f'Failed to add processor. Error: {e}', 'danger') + current_app.logger.error(f'Failed to add retriever {new_processor.name}' + f'for tenant {tenant_id}. Error: {str(e)}') + + return render_template('document/processor.html', form=form) + + +@document_bp.route('/processor/', methods=['GET', 'POST']) +@roles_accepted('Super User', 'Tenant Admin') +def edit_processor(processor_id): + """Edit an existing processorr configuration.""" + # Get the processor or return 404 + processor = Processor.query.get_or_404(processor_id) + + if processor.catalog_id: + # If catalog_id is just an ID, fetch the Catalog object + processor.catalog = Catalog.query.get(processor.catalog_id) + else: + processor.catalog = None + + # Create form instance with the processor + form = EditProcessorForm(request.form, obj=processor) + + configuration_config = PROCESSOR_TYPES[processor.type]["configuration"] + form.add_dynamic_fields("configuration", configuration_config, processor.configuration) + + if form.validate_on_submit(): + # Update basic fields + form.populate_obj(processor) + processor.configuration = form.get_dynamic_data('configuration') + + # Update catalog relationship + processor.catalog_id = form.catalog.data.id if form.catalog.data else None + + # Update logging information + update_logging_information(processor, dt.now(tz.utc)) + + # Save changes to database + try: + db.session.add(processor) + db.session.commit() + flash('Retriever updated successfully!', 'success') + current_app.logger.info(f'Retriever {processor.id} updated successfully') + except SQLAlchemyError as e: + db.session.rollback() + flash(f'Failed to update processor. Error: {str(e)}', 'danger') + current_app.logger.error(f'Failed to update processor {processor_id}. Error: {str(e)}') + return render_template('document/edit_processor.html', form=form, processor_id=processor_id) + + return redirect(prefixed_url_for('document_bp.processors')) + else: + form_validation_failed(request, form) + + return render_template('document/edit_processor.html', form=form, processor_id=processor_id) + + +@document_bp.route('/processors', methods=['GET', 'POST']) +@roles_accepted('Super User', 'Tenant Admin') +def processors(): + page = request.args.get('page', 1, type=int) + per_page = request.args.get('per_page', 10, type=int) + + query = Processor.query.order_by(Processor.id) + + pagination = query.paginate(page=page, per_page=per_page) + the_processors = pagination.items + + # prepare table data + rows = prepare_table_for_macro(the_processors, + [('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')]) + + # Render the catalogs in a template + return render_template('document/processors.html', rows=rows, pagination=pagination) + + +@document_bp.route('/handle_processor_selection', methods=['POST']) +@roles_accepted('Super User', 'Tenant Admin') +def handle_processor_selection(): + processor_identification = request.form.get('selected_row') + processor_id = ast.literal_eval(processor_identification).get('value') + action = request.form['action'] + + if action == 'edit_processor': + return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=processor_id)) + + return redirect(prefixed_url_for('document_bp.processors')) + + @document_bp.route('/retriever', methods=['GET', 'POST']) @roles_accepted('Super User', 'Tenant Admin') def retriever(): @@ -198,15 +278,14 @@ def retriever(): db.session.commit() flash('Retriever successfully added!', 'success') current_app.logger.info(f'Catalog {new_retriever.name} successfully added for tenant {tenant_id}!') + # Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type) + return redirect(prefixed_url_for('document_bp.edit_retriever', retriever_id=new_retriever.id)) except SQLAlchemyError as e: db.session.rollback() flash(f'Failed to add retriever. Error: {e}', 'danger') current_app.logger.error(f'Failed to add retriever {new_retriever.name}' f'for tenant {tenant_id}. Error: {str(e)}') - # Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type) - return redirect(prefixed_url_for('document_bp.retriever', retriever_id=new_retriever.id)) - return render_template('document/retriever.html', form=form) @@ -311,6 +390,7 @@ def add_document(): current_app.logger.info(f'Adding Document for {catalog_id}') tenant_id = session['tenant']['id'] file = form.file.data + sub_file_type = form.sub_file_type.data filename = secure_filename(file.filename) extension = filename.rsplit('.', 1)[1].lower() @@ -324,14 +404,13 @@ def add_document(): api_input = { 'catalog_id': catalog_id, 'name': form.name.data, + 'sub_file_type': form.sub_file_type.data, 'language': form.language.data, 'user_context': form.user_context.data, 'valid_from': form.valid_from.data, 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None, 'catalog_properties': catalog_properties, } - current_app.logger.debug(f'Creating document stack with input {api_input}') - new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id) task_id = start_embedding_task(tenant_id, new_doc_vers.id) @@ -341,6 +420,7 @@ def add_document(): except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e: flash(str(e), 'error') + current_app.logger.error(f"Error adding document: {str(e)}") except Exception as e: current_app.logger.error(f'Error adding document: {str(e)}') flash('An error occurred while adding the document.', 'error') @@ -378,6 +458,7 @@ def add_url(): api_input = { 'catalog_id': catalog_id, 'name': form.name.data or filename, + 'sub_file_type': form.sub_file_type.data, 'url': url, 'language': form.language.data, 'user_context': form.user_context.data, @@ -562,8 +643,6 @@ def handle_document_version_selection(): action = request.form['action'] - current_app.logger.debug(f'Triggered Document Version Action: {action}') - match action: case 'edit_document_version': return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=doc_vers_id)) @@ -598,9 +677,7 @@ def handle_library_selection(): @document_bp.route('/document_versions_list', methods=['GET']) @roles_accepted('Super User', 'Tenant Admin') def document_versions_list(): - current_app.logger.debug('Getting document versions list') view = DocumentVersionListView(DocumentVersion, 'document/document_versions_list_view.html', per_page=20) - current_app.logger.debug('Got document versions list') return view.get() @@ -653,8 +730,9 @@ def update_logging_information(obj, timestamp): def log_session_state(session, msg=""): - current_app.logger.debug(f"{msg} - Session dirty: {session.dirty}") - current_app.logger.debug(f"{msg} - Session new: {session.new}") + pass + # current_app.logger.info(f"{msg} - Session dirty: {session.dirty}") + # current_app.logger.info(f"{msg} - Session new: {session.new}") def fetch_html(url): diff --git a/eveai_app/views/dynamic_form_base.py b/eveai_app/views/dynamic_form_base.py index f518003..760de9e 100644 --- a/eveai_app/views/dynamic_form_base.py +++ b/eveai_app/views/dynamic_form_base.py @@ -5,6 +5,7 @@ import json from wtforms.fields.choices import SelectField from wtforms.fields.datetime import DateField +from common.utils.config_field_types import TaggingFields class DynamicFormBase(FlaskForm): @@ -38,14 +39,35 @@ class DynamicFormBase(FlaskForm): message=f"Value must be between {min_value or '-∞'} and {max_value or '∞'}" ) ) + elif field_type == 'tagging_fields': + validators_list.append(self._validate_tagging_fields) return validators_list + def _validate_tagging_fields(self, form, field): + """Validate the tagging fields structure""" + if not field.data: + return + + try: + # Parse JSON data + fields_data = json.loads(field.data) + + # Validate using TaggingFields model + try: + TaggingFields.from_dict(fields_data) + except ValueError as e: + raise ValidationError(str(e)) + + except json.JSONDecodeError: + raise ValidationError("Invalid JSON format") + except Exception as e: + raise ValidationError(f"Invalid field definition: {str(e)}") + def add_dynamic_fields(self, collection_name, config, initial_data=None): """Add dynamic fields to the form based on the configuration.""" self.dynamic_fields[collection_name] = [] for field_name, field_def in config.items(): - current_app.logger.debug(f"{field_name}: {field_def}") # Prefix the field name with the collection name full_field_name = f"{collection_name}_{field_name}" label = field_def.get('name', field_name) @@ -59,7 +81,6 @@ class DynamicFormBase(FlaskForm): # Handle special case for tagging_fields if field_type == 'tagging_fields': field_class = TextAreaField - field_validators.append(validate_tagging_fields) extra_classes = 'json-editor' field_kwargs = {} elif field_type == 'enum': @@ -145,16 +166,12 @@ class DynamicFormBase(FlaskForm): def get_dynamic_data(self, collection_name): """Retrieve the data from dynamic fields of a specific collection.""" data = {} - current_app.logger.debug(f"{collection_name} in {self.dynamic_fields}?") if collection_name not in self.dynamic_fields: return data prefix_length = len(collection_name) + 1 # +1 for the underscore for full_field_name in self.dynamic_fields[collection_name]: - current_app.logger.debug(f"{full_field_name}: {full_field_name}") original_field_name = full_field_name[prefix_length:] - current_app.logger.debug(f"{original_field_name}: {original_field_name}") field = getattr(self, full_field_name) - current_app.logger.debug(f"{field}: {field}") # Parse JSON for tagging_fields type if isinstance(field, TextAreaField) and field.data: try: diff --git a/eveai_app/views/entitlements_views.py b/eveai_app/views/entitlements_views.py index 962a6c9..449cb3c 100644 --- a/eveai_app/views/entitlements_views.py +++ b/eveai_app/views/entitlements_views.py @@ -159,7 +159,6 @@ def create_license(license_tier_id): tenant_id=tenant_id, tier_id=license_tier_id, ) - current_app.logger.debug(f"Currency data in form: {form.currency.data}") if form.validate_on_submit(): # Update the license with form data form.populate_obj(new_license) diff --git a/eveai_app/views/interaction_forms.py b/eveai_app/views/interaction_forms.py index f75d2ea..d63b867 100644 --- a/eveai_app/views/interaction_forms.py +++ b/eveai_app/views/interaction_forms.py @@ -33,7 +33,7 @@ class SpecialistForm(FlaskForm): type = SelectField('Specialist Type', validators=[DataRequired()]) - tuning = BooleanField('Enable Retrieval Tuning', default=False) + tuning = BooleanField('Enable Specialist Tuning', default=False) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/eveai_app/views/interaction_views.py b/eveai_app/views/interaction_views.py index 415e368..feefcf0 100644 --- a/eveai_app/views/interaction_views.py +++ b/eveai_app/views/interaction_views.py @@ -32,13 +32,11 @@ interaction_bp = Blueprint('interaction_bp', __name__, url_prefix='/interaction' @interaction_bp.before_request def log_before_request(): - current_app.logger.debug(f"Before request (interaction_bp): {request.method} {request.url}") + pass @interaction_bp.after_request def log_after_request(response): - current_app.logger.debug( - f"After request (interaction_bp): {request.method} {request.url} - Status: {response.status}") return response @@ -48,8 +46,6 @@ def before_request(): mw_before_request() except Exception as e: current_app.logger.error(f'Error switching schema in Interaction Blueprint: {e}') - for role in current_user.roles: - current_app.logger.debug(f'User {current_user.email} has role {role.name}') raise @@ -147,14 +143,9 @@ def specialist(): db.session.add(new_specialist) db.session.flush() # This assigns the ID to the specialist without committing the transaction - current_app.logger.debug( - f'New Specialist after flush - id: {new_specialist.id}, name: {new_specialist.name}') - # Create the retriever associations selected_retrievers = form.retrievers.data - current_app.logger.debug(f'Selected Retrievers - {selected_retrievers}') for retriever in selected_retrievers: - current_app.logger.debug(f'Creating association for Retriever - {retriever.id}') specialist_retriever = SpecialistRetriever( specialist_id=new_specialist.id, retriever_id=retriever.id @@ -174,7 +165,7 @@ def specialist(): flash(f'Failed to add specialist. Error: {str(e)}', 'danger') return render_template('interaction/specialist.html', form=form) - return render_template('interaction/specialists.html', form=form) + return render_template('interaction/specialist.html', form=form) @interaction_bp.route('/specialist/', methods=['GET', 'POST']) @@ -187,35 +178,31 @@ def edit_specialist(specialist_id): form.add_dynamic_fields("configuration", configuration_config, specialist.configuration) if request.method == 'GET': - # Pre-populate the retrievers field with current associations - current_app.logger.debug(f'Specialist retrievers: {specialist.retrievers}') - current_app.logger.debug(f'Form Retrievers Data Before: {form.retrievers.data}') - # Get the actual Retriever objects for the associated retriever_ids retriever_objects = Retriever.query.filter( Retriever.id.in_([sr.retriever_id for sr in specialist.retrievers]) ).all() form.retrievers.data = retriever_objects - current_app.logger.debug(f'Form Retrievers Data After: {form.retrievers.data}') - if form.validate_on_submit(): # Update the basic fields - form.populate_obj(specialist) + specialist.name = form.name.data + specialist.description = form.description.data + specialist.tuning = form.tuning.data # Update the configuration dynamic fields specialist.configuration = form.get_dynamic_data("configuration") - # Update retriever associations - current_retrievers = set(sr.retriever_id for sr in specialist.retrievers) - selected_retrievers = set(r.id for r in form.retrievers.data) + # Get current and selected retrievers + current_retrievers = {sr.retriever_id: sr for sr in specialist.retrievers} + selected_retrievers = {r.id: r for r in form.retrievers.data} # Remove unselected retrievers - for sr in specialist.retrievers[:]: - if sr.retriever_id not in selected_retrievers: - db.session.delete(sr) + for retriever_id in set(current_retrievers.keys()) - set(selected_retrievers.keys()): + specialist_retriever = current_retrievers[retriever_id] + db.session.delete(specialist_retriever) # Add new retrievers - for retriever_id in selected_retrievers - current_retrievers: + for retriever_id in set(selected_retrievers.keys()) - set(current_retrievers.keys()): specialist_retriever = SpecialistRetriever( specialist_id=specialist.id, retriever_id=retriever_id @@ -229,13 +216,12 @@ def edit_specialist(specialist_id): db.session.commit() flash('Specialist updated successfully!', 'success') current_app.logger.info(f'Specialist {specialist.id} updated successfully') + return redirect(prefixed_url_for('interaction_bp.specialists')) except SQLAlchemyError as e: db.session.rollback() flash(f'Failed to update specialist. Error: {str(e)}', 'danger') current_app.logger.error(f'Failed to update specialist {specialist_id}. Error: {str(e)}') return render_template('interaction/edit_specialist.html', form=form, specialist_id=specialist_id) - - return redirect(prefixed_url_for('interaction_bp.specialists')) else: form_validation_failed(request, form) diff --git a/eveai_app/views/security_views.py b/eveai_app/views/security_views.py index 9adfad7..94de540 100644 --- a/eveai_app/views/security_views.py +++ b/eveai_app/views/security_views.py @@ -22,20 +22,11 @@ security_bp = Blueprint('security_bp', __name__) @security_bp.before_request def log_before_request(): - current_app.logger.debug(f"Before request (security_bp): {request.method} {request.url}") - if current_user and current_user.is_authenticated: - current_app.logger.debug(f"After request (security_bp): Current User: {current_user.email}") - else: - current_app.logger.debug(f"After request (security_bp): No user logged in") + pass @security_bp.after_request def log_after_request(response): - current_app.logger.debug(f"After request (security_bp): {request.method} {request.url} - Status: {response.status}") - if current_user and current_user.is_authenticated: - current_app.logger.debug(f"After request (security_bp): Current User: {current_user.email}") - else: - current_app.logger.debug(f"After request (security_bp): No user logged in") return response @@ -47,13 +38,12 @@ def login(): form = LoginForm() if request.method == 'POST': - current_app.logger.debug(f"Starting login procedure for {form.email.data}") try: if form.validate_on_submit(): user = User.query.filter_by(email=form.email.data).first() if user is None or not verify_and_update_password(form.password.data, user): flash('Invalid username or password', 'danger') - current_app.logger.debug(f'Failed to login user') + current_app.logger.error(f'Failed to login user') return redirect(prefixed_url_for('security_bp.login')) if login_user(user): @@ -65,10 +55,10 @@ def login(): return redirect(prefixed_url_for('user_bp.tenant_overview')) else: flash('Invalid username or password', 'danger') - current_app.logger.debug(f'Failed to login user {user.email}') + current_app.logger.error(f'Failed to login user {user.email}') abort(401) else: - current_app.logger.debug(f'Invalid login form: {form.errors}') + current_app.logger.error(f'Invalid login form: {form.errors}') except CSRFError: current_app.logger.warning('CSRF token mismatch during login attempt') @@ -77,19 +67,14 @@ def login(): if request.method == 'GET': csrf_token = generate_csrf() - current_app.logger.debug(f'Generated new CSRF token: {csrf_token}') - # current_app.logger.debug(f"Login route completed - Session ID: {session.sid}") - current_app.logger.debug(f"Login route completed - Session data: {session}") return render_template('security/login_user.html', login_user_form=form) @security_bp.route('/logout', methods=['GET', 'POST']) @login_required def logout(): - current_app.logger.debug('Logging out') logout_user() - current_app.logger.debug('After Logout') return redirect(prefixed_url_for('basic_bp.index')) @@ -99,17 +84,13 @@ def confirm_email(token): email = confirm_token(token) except Exception as e: flash('The confirmation link is invalid or has expired.', 'danger') - current_app.logger.debug(f'Invalid confirmation link detected: {token} - error: {e}') return redirect(prefixed_url_for('basic_bp.confirm_email_fail')) user = User.query.filter_by(email=email).first_or_404() - current_app.logger.debug(f'Trying to confirm email for user {user.email}') if user.active: flash('Account already confirmed. Please login.', 'success') - current_app.logger.debug(f'Account for user {user.email} was already activated') return redirect(prefixed_url_for('security_bp.login')) else: - current_app.logger.debug(f'Trying to confirm email for user {user.email}') user.active = True user.updated_at = dt.now(tz.utc) user.confirmed_at = dt.now(tz.utc) @@ -119,10 +100,8 @@ def confirm_email(token): db.session.commit() except SQLAlchemyError as e: db.session.rollback() - current_app.logger.debug(f'Failed to confirm email for user {user.email}: {e}') return redirect(prefixed_url_for('basic_bp.confirm_email_fail')) - current_app.logger.debug(f'Account for user {user.email} was confirmed.') send_reset_email(user) return redirect(prefixed_url_for('basic_bp.confirm_email_ok')) @@ -145,7 +124,7 @@ def reset_password(token): email = confirm_token(token) except Exception as e: flash('The reset link is invalid or has expired.', 'danger') - current_app.logger.debug(f'Invalid reset link detected: {token} - error: {e}') + current_app.logger.error(f'Invalid reset link detected: {token} - error: {e}') return redirect(prefixed_url_for('security_bp.reset_password_request')) user = User.query.filter_by(email=email).first_or_404() diff --git a/eveai_app/views/user_views.py b/eveai_app/views/user_views.py index 523c8bb..f669967 100644 --- a/eveai_app/views/user_views.py +++ b/eveai_app/views/user_views.py @@ -21,20 +21,11 @@ user_bp = Blueprint('user_bp', __name__, url_prefix='/user') @user_bp.before_request def log_before_request(): - current_app.logger.debug(f"Before request (user_bp): {request.method} {request.url}") - if current_user and current_user.is_authenticated: - current_app.logger.debug(f"After request (user_bp): Current User: {current_user.email}") - else: - current_app.logger.debug(f"After request (user_bp): No user logged in") + pass @user_bp.after_request def log_after_request(response): - current_app.logger.debug(f"After request (user_bp): {request.method} {request.url} - Status: {response.status}") - if current_user and current_user.is_authenticated: - current_app.logger.debug(f"After request (user_bp): Current User: {current_user.email}") - else: - current_app.logger.debug(f"After request (user_bp): No user logged in") return response @@ -43,7 +34,6 @@ def log_after_request(response): def tenant(): form = TenantForm() if form.validate_on_submit(): - current_app.logger.debug('Creating new tenant') # Handle the required attributes new_tenant = Tenant() form.populate_obj(new_tenant) @@ -91,7 +81,6 @@ def edit_tenant(tenant_id): form.populate_obj(tenant) if form.validate_on_submit(): - current_app.logger.debug(f'Updating tenant {tenant_id}') # Populate the tenant with form data form.populate_obj(tenant) @@ -102,7 +91,6 @@ def edit_tenant(tenant_id): session['tenant'] = tenant.to_dict() # return redirect(url_for(f"user/tenant/tenant_id")) else: - current_app.logger.debug(f'Tenant update failed with errors: {form.errors}') form_validation_failed(request, form) return render_template('user/tenant.html', form=form, tenant_id=tenant_id) @@ -142,7 +130,7 @@ def user(): # security.datastore.set_uniquifier(new_user) try: send_confirmation_email(new_user) - current_app.logger.debug(f'User {new_user.id} with name {new_user.user_name} added to database' + current_app.logger.info(f'User {new_user.id} with name {new_user.user_name} added to database' f'Confirmation email sent to {new_user.email}') flash('User added successfully and confirmation email sent.', 'success') except Exception as e: @@ -448,11 +436,7 @@ def generate_api_api_key(): @user_bp.route('/tenant_overview', methods=['GET']) @roles_accepted('Super User', 'Tenant Admin') def tenant_overview(): - current_app.logger.debug('Rendering tenant overview') - current_app.logger.debug(f'current_user: {current_user}') - current_app.logger.debug(f'Current user roles: {current_user.roles}') tenant_id = session['tenant']['id'] - current_app.logger.debug(f'Generating overview for tenant {tenant_id}') tenant = Tenant.query.get_or_404(tenant_id) form = TenantForm(obj=tenant) return render_template('user/tenant_overview.html', form=form) diff --git a/eveai_chat/socket_handlers/chat_handler.py b/eveai_chat/socket_handlers/chat_handler.py index b367063..2e9f697 100644 --- a/eveai_chat/socket_handlers/chat_handler.py +++ b/eveai_chat/socket_handlers/chat_handler.py @@ -38,7 +38,6 @@ def track_socketio_event(func): @track_socketio_event def handle_connect(): try: - current_app.logger.debug(f'SocketIO: Connection handling started using {request.args}') tenant_id = request.args.get('tenantId') if not tenant_id: raise Exception("Missing Tenant ID") @@ -52,12 +51,10 @@ def handle_connect(): # Create JWT token token = create_access_token(identity={"tenant_id": tenant_id, "api_key": api_key}) - current_app.logger.debug(f'SocketIO: Connection handling created token: {token} for tenant {tenant_id}') # Create a unique room for this client room = f"{tenant_id}_{request.sid}" join_room(room) - current_app.logger.debug(f'SocketIO: Client joined room: {room}') # Create a unique session ID if 'session_id' not in session: @@ -67,11 +64,8 @@ def handle_connect(): session['room'] = room # Communicate connection to client - current_app.logger.debug(f'SocketIO: Connection handling sending status to client for tenant {tenant_id}') emit('connect', {'status': 'Connected', 'tenant_id': tenant_id, 'room': room}) - current_app.logger.debug(f'SocketIO: Connection handling sending authentication token to client') emit('authenticated', {'token': token, 'room': room}) # Emit custom event with the token - current_app.logger.debug(f'SocketIO: Connection handling sent token to client for tenant {tenant_id}') except Exception as e: current_app.logger.error(f'SocketIO: Connection failed: {e}') # communicate connection problem to client @@ -85,71 +79,60 @@ def handle_disconnect(): room = session.get('room') if room: leave_room(room) - current_app.logger.debug(f'SocketIO: Client left room: {room}') - current_app.logger.debug('SocketIO: Client disconnected') @socketio.on('heartbeat') def handle_heartbeat(): - current_app.logger.debug('SocketIO: Heartbeat received') last_activity = session.get('last_activity') if datetime.now() - last_activity > current_app.config.get('SOCKETIO_MAX_IDLE_TIME'): - current_app.logger.debug('SocketIO: Heartbeat timed out, connection closed') disconnect() @socketio.on('user_message') def handle_message(data): try: - current_app.logger.debug(f"SocketIO: Message handling received message from tenant {data['tenantId']}: " - f"{data['message']} with token {data['token']}") session['last_activity'] = datetime.now() - current_tenant_id = validate_incoming_data(data) room = session.get('room') # Offload actual processing of question - task = current_celery.send_task('ask_question', + task = current_celery.send_task('execute_specialist', queue='llm_interactions', args=[ current_tenant_id, - data['message'], - data['language'], + data['specialistId'], + data['arguments'], session['session_id'], data['timezone'], room ]) - current_app.logger.debug(f'SocketIO: Message offloading for tenant {current_tenant_id}, ' - f'Question: {task.id}') response = { 'tenantId': data['tenantId'], 'message': f'Processing question ... Session ID = {session["session_id"]}', 'taskId': task.id, } - current_app.logger.debug(f"SocketIO: Message handling sent bot response: {response}") + current_app.logger.debug(f"Sent message with {data}, response {response}") emit('bot_response', response, room=room) except Exception as e: - current_app.logger.error(f'SocketIO: Message handling failed: {e}') + current_app.logger.error(f'SocketIO: Message handling failed: {str(e)}') disconnect() @socketio.on('check_task_status') def check_task_status(data): + current_app.logger.debug(f'SocketIO: Checking Task Status ... {data}') task_id = data.get('task_id') room = session.get('room') - current_app.logger.debug(f'SocketIO: Check task status for task_id: {task_id}') if not task_id: emit('task_status', {'status': 'error', 'message': 'Missing task ID'}, room=room) return task_result = current_celery.AsyncResult(task_id) if task_result.state == 'PENDING': - current_app.logger.debug(f'SocketIO: Task {task_id} is pending') emit('task_status', {'status': 'pending', 'taskId': task_id}, room=room) elif task_result.state == 'SUCCESS': - current_app.logger.debug(f'SocketIO: Task {task_id} has finished. Status: {task_result.state}, ' - f'Result: {task_result.result}') result = task_result.result + current_app.logger.debug(f'SocketIO: Task {task_id} returned: {result}') response = { 'status': 'success', 'taskId': task_id, @@ -167,8 +150,6 @@ def check_task_status(data): @socketio.on('feedback') def handle_feedback(data): try: - current_app.logger.debug(f'SocketIO: Feedback handling received feedback with data: {data}') - current_tenant_id = validate_incoming_data(data) interaction_id = data.get('interactionId') @@ -177,7 +158,6 @@ def handle_feedback(data): Database(current_tenant_id).switch_schema() interaction = Interaction.query.get_or_404(interaction_id) - current_app.logger.debug(f'Processing feedback for interaction: {interaction}') interaction.appreciation = 0 if feedback == 'down' else 100 try: db.session.commit() @@ -188,7 +168,7 @@ def handle_feedback(data): emit('feedback_received', {'status': 'Could not register feedback', 'interaction_id': interaction_id}) raise e except Exception as e: - current_app.logger.debug(f'SocketIO: Feedback handling failed: {e}') + current_app.logger.error(f'SocketIO: Feedback handling failed: {e}') disconnect() @@ -212,7 +192,6 @@ def validate_incoming_data(data): if not token_sub: raise Exception("Missing token subject") - tenant_id = token_sub.get('tenant_id') current_tenant_id = token_sub.get('tenant_id') if not current_tenant_id: diff --git a/eveai_chat_workers/__init__.py b/eveai_chat_workers/__init__.py index 5cf4bb7..4aaf082 100644 --- a/eveai_chat_workers/__init__.py +++ b/eveai_chat_workers/__init__.py @@ -3,11 +3,14 @@ import logging.config from flask import Flask import os +from common.langchain.templates.template_manager import TemplateManager from common.utils.celery_utils import make_celery, init_celery -from common.extensions import db +from common.extensions import db, template_manager, cache_manager from config.logging_config import LOGGING from config.config import get_config +from . import specialists, retrievers + def create_app(config_file=None): app = Flask(__name__) @@ -24,14 +27,12 @@ def create_app(config_file=None): logging.config.dictConfig(LOGGING) - app.logger.debug('Starting up eveai_chat_workers...') + app.logger.info('Starting up eveai_chat_workers...') register_extensions(app) celery = make_celery(app.name, app.config) init_celery(celery, app) - app.rag_tuning_logger = logging.getLogger('rag_tuning') - from eveai_chat_workers import tasks print(tasks.tasks_ping()) @@ -40,6 +41,9 @@ def create_app(config_file=None): def register_extensions(app): db.init_app(app) + cache_manager.init_app(app) + template_manager.init_app(app) app, celery = create_app() + diff --git a/eveai_chat_workers/chat_session_cache.py b/eveai_chat_workers/chat_session_cache.py new file mode 100644 index 0000000..83a2512 --- /dev/null +++ b/eveai_chat_workers/chat_session_cache.py @@ -0,0 +1,193 @@ +# common/utils/cache/chat_session_handler.py +from typing import Dict, List, Any, Optional +from datetime import datetime as dt, timezone as tz +from dataclasses import dataclass + +from flask import current_app +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import joinedload + +from common.extensions import db, cache_manager +from common.models.interaction import ChatSession, Interaction +from common.utils.cache.base import CacheHandler + + +@dataclass +class CachedInteraction: + """Lightweight representation of an interaction for history purposes""" + specialist_arguments: Dict[str, Any] # Contains the original question and other arguments + specialist_results: Dict[str, Any] # Contains detailed question, answer and other results + + +@dataclass +class CachedSession: + """Cached representation of a chat session with its interactions""" + id: int + session_id: str + interactions: List[CachedInteraction] + timezone: str + + +class ChatSessionCacheHandler(CacheHandler[CachedSession]): + """Handles caching of chat sessions focused on interaction history""" + handler_name = 'chat_session_cache' + + def __init__(self, region): + super().__init__(region, 'chat_session') + self.configure_keys('session_id') + + def get_cached_session(self, session_id: str, *, create_params: Optional[Dict[str, Any]] = None) -> CachedSession: + """ + Get or create a cached session with its interaction history. + If not in cache, loads from database and caches it. + + Args: + session_id: The session identifier + create_params: Optional parameters for session creation if it doesn't exist. + Must include 'timezone' if provided. + + + Returns: + CachedSession with interaction history + + """ + + def creator_func(session_id: str) -> CachedSession: + # Load session and interactions from database + session = ( + ChatSession.query + .options(joinedload(ChatSession.interactions)) + .filter_by(session_id=session_id) + .first() + ) + + if not session: + if not create_params: + raise ValueError(f"Chat session {session_id} not found and no creation parameters provided") + + if 'timezone' not in create_params: + raise ValueError("timezone is required in create_params for new session creation") + + # Create new session + session = ChatSession( + session_id=session_id, + session_start=dt.now(tz.utc), + timezone=create_params['timezone'] + ) + try: + db.session.add(session) + db.session.commit() + except SQLAlchemyError as e: + db.session.rollback() + raise ValueError(f"Failed to create new session: {str(e)}") + + # Convert to cached format + cached_interactions = [ + CachedInteraction( + specialist_arguments=interaction.specialist_arguments, + specialist_results=interaction.specialist_results + ) + for interaction in session.interactions + if interaction.specialist_results is not None # Only include completed interactions + ] + + return CachedSession( + id=session.id, + session_id=session_id, + interactions=cached_interactions, + timezone=session.timezone + ) + + return self.get(creator_func, session_id=session_id) + + def add_completed_interaction(self, session_id: str, interaction: Interaction) -> None: + """ + Add a completed interaction to the cached session history. + Should only be called once the interaction has an answer. + + Args: + session_id: The session identifier + interaction: The completed interaction to add + + Note: + Only adds the interaction if it has an answer + """ + if not interaction.specialist_results: + return # Skip incomplete interactions + + try: + cached_session = self.get_cached_session(session_id) + + # Add new interaction to cache + cached_session.interactions.append( + CachedInteraction( + specialist_arguments=interaction.specialist_arguments, + specialist_results=interaction.specialist_results, + ) + ) + + # Force cache update + self.invalidate(session_id=session_id) + + except ValueError: + # If session not in cache yet, load it fresh from DB + self.get_cached_session(session_id) + + def to_cache_data(self, instance: CachedSession) -> Dict[str, Any]: + """Convert CachedSession to cache data""" + return { + 'id': instance.id, + 'session_id': instance.session_id, + 'timezone': instance.timezone, + 'interactions': [ + { + 'specialist_arguments': interaction.specialist_arguments, + 'specialist_results': interaction.specialist_results, + } + for interaction in instance.interactions + ], + 'last_updated': dt.now(tz=tz.utc).isoformat() + } + + def from_cache_data(self, data: Dict[str, Any], session_id: str, **kwargs) -> CachedSession: + """Create CachedSession from cache data""" + interactions = [ + CachedInteraction( + specialist_arguments=int_data['specialist_arguments'], + specialist_results=int_data['specialist_results'] + ) + for int_data in data['interactions'] + ] + + return CachedSession( + id=data['id'], + session_id=data['session_id'], + interactions=interactions, + timezone=data['timezone'] + ) + + def should_cache(self, value: Dict[str, Any]) -> bool: + """Validate cache data""" + required_fields = {'id','session_id', 'timezone', 'interactions'} + return all(field in value for field in required_fields) + + +# Register the handler with the cache manager +cache_manager.register_handler(ChatSessionCacheHandler, 'eveai_chat_workers') + + +# Helper function similar to get_model_variables +def get_chat_history(session_id: str) -> CachedSession: + """ + Get cached chat history for a session, loading from database if needed + + Args: + session_id: Session ID to look up + + Returns: + CachedSession with interaction history + + Raises: + ValueError: If session doesn't exist + """ + return cache_manager.chat_session_cache.get_cached_session(session_id) \ No newline at end of file diff --git a/eveai_chat_workers/retrievers/__init__.py b/eveai_chat_workers/retrievers/__init__.py new file mode 100644 index 0000000..37bd40e --- /dev/null +++ b/eveai_chat_workers/retrievers/__init__.py @@ -0,0 +1,5 @@ +# Import all specialist implementations here to ensure registration +from . import standard_rag + +# List of all available specialist implementations +__all__ = ['standard_rag'] \ No newline at end of file diff --git a/eveai_chat_workers/retrievers/base.py b/eveai_chat_workers/retrievers/base.py new file mode 100644 index 0000000..8c40b1d --- /dev/null +++ b/eveai_chat_workers/retrievers/base.py @@ -0,0 +1,57 @@ +from abc import ABC, abstractmethod, abstractproperty +from typing import Dict, Any, List + +from flask import current_app +from eveai_chat_workers.retrievers.retriever_typing import RetrieverResult, RetrieverArguments +from config.logging_config import TuningLogger + + +class BaseRetriever(ABC): + """Base class for all retrievers""" + + def __init__(self, tenant_id: int, retriever_id: int): + self.tenant_id = tenant_id + self.retriever_id = retriever_id + self.tuning = False + self.tuning_logger = None + self._setup_tuning_logger() + + @property + @abstractmethod + def type(self) -> str: + """The type of the retriever""" + pass + + def _setup_tuning_logger(self): + try: + self.tuning_logger = TuningLogger( + 'tuning', + tenant_id=self.tenant_id, + retriever_id=self.retriever_id, + ) + # Verify logger is working with a test message + if self.tuning: + self.tuning_logger.log_tuning('retriever', "Tuning logger initialized") + except Exception as e: + current_app.logger.error(f"Failed to setup tuning logger: {str(e)}") + raise + + def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None: + if self.tuning and self.tuning_logger: + try: + self.tuning_logger.log_tuning('retriever', message, data) + except Exception as e: + current_app.logger.error(f"Processor: Error in tuning logging: {e}") + + @abstractmethod + def retrieve(self, arguments: RetrieverArguments) -> List[RetrieverResult]: + """ + Retrieve relevant documents based on provided arguments + + Args: + arguments: Dictionary of arguments for the retrieval operation + + Returns: + List[Dict[str, Any]]: List of retrieved documents/content + """ + pass diff --git a/eveai_chat_workers/retrievers/registry.py b/eveai_chat_workers/retrievers/registry.py new file mode 100644 index 0000000..d8a90eb --- /dev/null +++ b/eveai_chat_workers/retrievers/registry.py @@ -0,0 +1,20 @@ +from typing import Dict, Type +from .base import BaseRetriever + + +class RetrieverRegistry: + """Registry for retriever types""" + + _registry: Dict[str, Type[BaseRetriever]] = {} + + @classmethod + def register(cls, retriever_type: str, retriever_class: Type[BaseRetriever]): + """Register a new retriever type""" + cls._registry[retriever_type] = retriever_class + + @classmethod + def get_retriever_class(cls, retriever_type: str) -> Type[BaseRetriever]: + """Get the retriever class for a given type""" + if retriever_type not in cls._registry: + raise ValueError(f"Unknown retriever type: {retriever_type}") + return cls._registry[retriever_type] diff --git a/eveai_chat_workers/retrievers/retriever_typing.py b/eveai_chat_workers/retrievers/retriever_typing.py new file mode 100644 index 0000000..f5355c0 --- /dev/null +++ b/eveai_chat_workers/retrievers/retriever_typing.py @@ -0,0 +1,60 @@ +from typing import List, Dict, Any +from pydantic import BaseModel, Field, model_validator +from common.utils.config_field_types import ArgumentDefinition, TaggingFields +from config.retriever_types import RETRIEVER_TYPES + + +class RetrieverMetadata(BaseModel): + """Metadata structure for retrieved documents""" + document_id: int = Field(..., description="ID of the source document") + version_id: int = Field(..., description="Version ID of the source document") + document_name: str = Field(..., description="Name of the source document") + user_metadata: Dict[str, Any] = Field( + default_factory=dict, # This will use an empty dict if None is provided + description="User-defined metadata" + ) + + +class RetrieverResult(BaseModel): + """Standard result format for all retrievers""" + id: int = Field(..., description="ID of the retrieved embedding") + chunk: str = Field(..., description="Retrieved text chunk") + similarity: float = Field(..., description="Similarity score (0-1)") + metadata: RetrieverMetadata = Field(..., description="Associated metadata") + + +class RetrieverArguments(BaseModel): + """ + Dynamic arguments for retrievers, allowing arbitrary fields but validating required ones + based on RETRIEVER_TYPES configuration. + """ + type: str = Field(..., description="Type of retriever (e.g. STANDARD_RAG)") + + # Allow any additional fields + model_config = { + "extra": "allow" + } + + @model_validator(mode='after') + def validate_required_arguments(self) -> 'RetrieverArguments': + """Validate that all required arguments for this retriever type are present""" + retriever_config = RETRIEVER_TYPES.get(self.type) + if not retriever_config: + raise ValueError(f"Unknown retriever type: {self.type}") + + # Check required arguments from configuration + for arg_name, arg_config in retriever_config['arguments'].items(): + if arg_config.get('required', False): + if not hasattr(self, arg_name): + raise ValueError(f"Missing required argument '{arg_name}' for {self.type}") + + # Type validation + value = getattr(self, arg_name) + expected_type = arg_config['type'] + if expected_type == 'str' and not isinstance(value, str): + raise ValueError(f"Argument '{arg_name}' must be a string") + elif expected_type == 'int' and not isinstance(value, int): + raise ValueError(f"Argument '{arg_name}' must be an integer") + # Add other type validations as needed + + return self diff --git a/eveai_chat_workers/retrievers/standard_rag.py b/eveai_chat_workers/retrievers/standard_rag.py new file mode 100644 index 0000000..ac5b877 --- /dev/null +++ b/eveai_chat_workers/retrievers/standard_rag.py @@ -0,0 +1,140 @@ +# retrievers/standard_rag.py +from datetime import datetime as dt, timezone as tz +from typing import Dict, Any, List +from sqlalchemy import func, or_, desc +from sqlalchemy.exc import SQLAlchemyError +from flask import current_app + +from common.extensions import db +from common.models.document import Document, DocumentVersion, Catalog, Retriever +from common.models.user import Tenant +from common.utils.datetime_utils import get_date_in_timezone +from common.utils.model_utils import get_model_variables +from .base import BaseRetriever + +from .registry import RetrieverRegistry +from .retriever_typing import RetrieverArguments, RetrieverResult, RetrieverMetadata + + +class StandardRAGRetriever(BaseRetriever): + """Standard RAG retriever implementation""" + + def __init__(self, tenant_id: int, retriever_id: int): + super().__init__(tenant_id, retriever_id) + + retriever = Retriever.query.get_or_404(retriever_id) + self.catalog_id = retriever.catalog_id + self.similarity_threshold = retriever.configuration.get('es_similarity_threshold', 0.3) + self.k = retriever.configuration.get('es_k', 8) + self.tuning = retriever.tuning + self.model_variables = get_model_variables(self.tenant_id) + + self._log_tuning("Standard RAG retriever initialized") + + @property + def type(self) -> str: + return "STANDARD_RAG" + + def retrieve(self, arguments: RetrieverArguments) -> List[RetrieverResult]: + """ + Retrieve documents based on query + + Args: + arguments: Validated RetrieverArguments containing at minimum: + - query: str - The search query + + Returns: + List[RetrieverResult]: List of retrieved documents with similarity scores + """ + try: + query = arguments.query + + # Get query embedding + query_embedding = self._get_query_embedding(query) + + # Get the appropriate embedding database model + db_class = self.model_variables.embedding_model_class + + # Get current date for validity checks + current_date = dt.now(tz=tz.utc).date() + + # Create subquery for latest versions + subquery = ( + db.session.query( + DocumentVersion.doc_id, + func.max(DocumentVersion.id).label('latest_version_id') + ) + .group_by(DocumentVersion.doc_id) + .subquery() + ) + + # Main query + query_obj = ( + db.session.query( + db_class, + (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity') + ) + .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id) + .join(Document, DocumentVersion.doc_id == Document.id) + .join(subquery, DocumentVersion.id == subquery.c.latest_version_id) + .filter( + or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date), + or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date), + (1 - db_class.embedding.cosine_distance(query_embedding)) > self.similarity_threshold, + Document.catalog_id == self.catalog_id + ) + .order_by(desc('similarity')) + .limit(self.k) + ) + + results = query_obj.all() + + # Transform results into standard format + processed_results = [] + for doc, similarity in results: + processed_results.append( + RetrieverResult( + id=doc.id, + chunk=doc.chunk, + similarity=float(similarity), + metadata=RetrieverMetadata( + document_id=doc.document_version.doc_id, + version_id=doc.document_version.id, + document_name=doc.document_version.document.name, + user_metadata=doc.document_version.user_metadata or {}, + ) + ) + ) + + # Log the retrieval + if self.tuning: + compiled_query = str(query_obj.statement.compile( + compile_kwargs={"literal_binds": True} # This will include the actual values in the SQL + )) + self._log_tuning('retrieve', { + "arguments": arguments.model_dump(), + "similarity_threshold": self.similarity_threshold, + "k": self.k, + "query": compiled_query, + "Raw Results": str(results), + "Processed Results": [r.model_dump() for r in processed_results], + }) + + return processed_results + + except SQLAlchemyError as e: + current_app.logger.error(f'Error in RAG retrieval: {e}') + db.session.rollback() + raise + except Exception as e: + current_app.logger.error(f'Unexpected error in RAG retrieval: {e}') + raise + + def _get_query_embedding(self, query: str): + """Get embedding for the query text""" + embedding_model = self.model_variables.embedding_model + return embedding_model.embed_query(query) + + +# Register the retriever type +RetrieverRegistry.register("STANDARD_RAG", StandardRAGRetriever) diff --git a/eveai_chat_workers/specialists/__init__.py b/eveai_chat_workers/specialists/__init__.py new file mode 100644 index 0000000..13b8d3f --- /dev/null +++ b/eveai_chat_workers/specialists/__init__.py @@ -0,0 +1,5 @@ +# Import all specialist implementations here to ensure registration +from . import rag_specialist + +# List of all available specialist implementations +__all__ = ['rag_specialist'] \ No newline at end of file diff --git a/eveai_chat_workers/specialists/base.py b/eveai_chat_workers/specialists/base.py new file mode 100644 index 0000000..86c45f3 --- /dev/null +++ b/eveai_chat_workers/specialists/base.py @@ -0,0 +1,50 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any +from flask import current_app + +from config.logging_config import TuningLogger +from eveai_chat_workers.specialists.specialist_typing import SpecialistArguments, SpecialistResult + + +class BaseSpecialist(ABC): + """Base class for all specialists""" + + def __init__(self, tenant_id: int, specialist_id: int, session_id: str): + self.tenant_id = tenant_id + self.specialist_id = specialist_id + self.session_id = session_id + self.tuning = False + self.tuning_logger = None + self._setup_tuning_logger() + + @property + @abstractmethod + def type(self) -> str: + """The type of the specialist""" + pass + + def _setup_tuning_logger(self): + try: + self.tuning_logger = TuningLogger( + 'tuning', + tenant_id=self.tenant_id, + specialist_id=self.specialist_id, + ) + # Verify logger is working with a test message + if self.tuning: + self.tuning_logger.log_tuning('specialist', "Tuning logger initialized") + except Exception as e: + current_app.logger.error(f"Failed to setup tuning logger: {str(e)}") + raise + + def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None: + if self.tuning and self.tuning_logger: + try: + self.tuning_logger.log_tuning('specialist', message, data) + except Exception as e: + current_app.logger.error(f"Processor: Error in tuning logging: {e}") + + @abstractmethod + def execute(self, arguments: SpecialistArguments) -> SpecialistResult: + """Execute the specialist's logic""" + pass diff --git a/eveai_chat_workers/specialists/rag_specialist.py b/eveai_chat_workers/specialists/rag_specialist.py new file mode 100644 index 0000000..a1ea5b7 --- /dev/null +++ b/eveai_chat_workers/specialists/rag_specialist.py @@ -0,0 +1,289 @@ +from datetime import datetime +from typing import Dict, Any, List +from flask import current_app +from langchain_core.exceptions import LangChainException +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnableParallel, RunnablePassthrough + +from common.langchain.outputs.base import OutputRegistry +from common.langchain.outputs.rag import RAGOutput +from common.utils.business_event_context import current_event +from .specialist_typing import SpecialistArguments, SpecialistResult +from ..chat_session_cache import CachedInteraction, get_chat_history +from ..retrievers.registry import RetrieverRegistry +from ..retrievers.base import BaseRetriever +from common.models.interaction import SpecialistRetriever, Specialist +from common.utils.model_utils import get_model_variables, create_language_template, replace_variable_in_template +from .base import BaseSpecialist +from .registry import SpecialistRegistry +from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments, RetrieverResult + + +class RAGSpecialist(BaseSpecialist): + """ + Standard Q&A RAG Specialist implementation that combines retriever results + with LLM processing to generate answers. + """ + def __init__(self, tenant_id: int, specialist_id: int, session_id: str): + super().__init__(tenant_id, specialist_id, session_id) + + # Check and load the specialist + specialist = Specialist.query.get_or_404(specialist_id) + # Set the specific configuration for the RAG Specialist + self.specialist_context = specialist.configuration.get('specialist_context', '') + self.temperature = specialist.configuration.get('temperature', 0.3) + self.tuning = specialist.tuning + + # Initialize retrievers + self.retrievers = self._initialize_retrievers() + + # Initialize model variables + self.model_variables = get_model_variables(tenant_id) + + @property + def type(self) -> str: + return "STANDARD_RAG" + + def _initialize_retrievers(self) -> List[BaseRetriever]: + """Initialize all retrievers associated with this specialist""" + retrievers = [] + + # Get retriever associations from database + specialist_retrievers = ( + SpecialistRetriever.query + .filter_by(specialist_id=self.specialist_id) + .all() + ) + + self._log_tuning("_initialize_retrievers", {"Nr of retrievers": len(specialist_retrievers)}) + + for spec_retriever in specialist_retrievers: + # Get retriever configuration from database + retriever = spec_retriever.retriever + retriever_class = RetrieverRegistry.get_retriever_class(retriever.type) + self._log_tuning("_initialize_retrievers", { + "Retriever id": spec_retriever.retriever_id, + "Retriever Type": retriever.type, + "Retriever Class": str(retriever_class), + }) + + # Initialize retriever with its configuration + retrievers.append( + retriever_class( + tenant_id=self.tenant_id, + retriever_id=retriever.id, + ) + ) + + return retrievers + + @property + def required_templates(self) -> List[str]: + """List of required templates for this specialist""" + return ['rag', 'history'] + + # def _detail_question(question, language, model_variables, session_id): + # retriever = EveAIHistoryRetriever(model_variables=model_variables, session_id=session_id) + # llm = model_variables['llm'] + # template = model_variables['history_template'] + # language_template = create_language_template(template, language) + # full_template = replace_variable_in_template(language_template, "{tenant_context}", + # model_variables['rag_context']) + # history_prompt = ChatPromptTemplate.from_template(full_template) + # setup_and_retrieval = RunnableParallel({"history": retriever, "question": RunnablePassthrough()}) + # output_parser = StrOutputParser() + # + # chain = setup_and_retrieval | history_prompt | llm | output_parser + # + # try: + # answer = chain.invoke(question) + # return answer + # except LangChainException as e: + # current_app.logger.error(f'Error detailing question: {e}') + # raise + + def _detail_question(self, language: str, question: str) -> str: + """Detail question based on conversation history""" + try: + # Get cached session history + cached_session = get_chat_history(self.session_id) + + # Format history for the prompt + formatted_history = "\n\n".join([ + f"HUMAN:\n{interaction.specialist_results.get('detailed_query')}\n\n" + f"AI:\n{interaction.specialist_results.get('answer')}" + for interaction in cached_session.interactions + ]) + + # Get LLM and template + llm = self.model_variables.get_llm(temperature=0.3) + template = self.model_variables.get_template('history') + language_template = create_language_template(template, language) + + # Create prompt + history_prompt = ChatPromptTemplate.from_template(language_template) + + # Create chain + chain = ( + history_prompt | + llm | + StrOutputParser() + ) + + # Execute chain + detailed_question = chain.invoke({ + "history": formatted_history, + "question": question + }) + + if self.tuning: + self._log_tuning("_detail_question", { + "cached_session_id": cached_session.session_id, + "cached_session.interactions": str(cached_session.interactions), + "original_question": question, + "history_used": formatted_history, + "detailed_question": detailed_question, + }) + + return detailed_question + + except Exception as e: + current_app.logger.error(f"Error detailing question: {e}") + return question # Fallback to original question + + def execute(self, arguments: SpecialistArguments) -> SpecialistResult: + """ + Execute the RAG specialist to generate an answer + """ + start_time = datetime.now() + + try: + with current_event.create_span("Specialist Detail Question"): + # Get required arguments + language = arguments.language + query = arguments.query + detailed_question = self._detail_question(language, query) + + # Log the start of retrieval process if tuning is enabled + with current_event.create_span("Specialist Retrieval"): + self._log_tuning("Starting context retrieval", { + "num_retrievers": len(self.retrievers), + "all arguments": arguments.model_dump(), + }) + + # Get retriever-specific arguments + retriever_arguments = arguments.retriever_arguments + + # Collect context from all retrievers + all_context = [] + for retriever in self.retrievers: + # Get arguments for this specific retriever + retriever_id = str(retriever.retriever_id) + if retriever_id not in retriever_arguments: + current_app.logger.error(f"Missing arguments for retriever {retriever_id}") + continue + + # Get the retriever's arguments and update the query + current_retriever_args = retriever_arguments[retriever_id] + if isinstance(retriever_arguments[retriever_id], RetrieverArguments): + updated_args = current_retriever_args.model_dump() + updated_args['query'] = detailed_question + retriever_args = RetrieverArguments(**updated_args) + else: + # Create a new RetrieverArguments instance from the dictionary + current_retriever_args['query'] = detailed_question + retriever_args = RetrieverArguments(**current_retriever_args) + + # Each retriever gets its own specific arguments + retriever_result = retriever.retrieve(retriever_args) + all_context.extend(retriever_result) + + # Sort by similarity if available and get unique contexts + all_context.sort(key=lambda x: x.similarity, reverse=True) + unique_contexts = [] + seen_chunks = set() + for ctx in all_context: + if ctx.chunk not in seen_chunks: + unique_contexts.append(ctx) + seen_chunks.add(ctx.chunk) + + self._log_tuning("Context retrieval completed", { + "total_contexts": len(all_context), + "unique_contexts": len(unique_contexts), + "average_similarity": sum(ctx.similarity for ctx in unique_contexts) / len( + unique_contexts) if unique_contexts else 0 + }) + + # Prepare context for LLM + formatted_context = "\n\n".join([ + f"SOURCE: {ctx.metadata.document_id}\n\n{ctx.chunk}" + for ctx in unique_contexts + ]) + + with current_event.create_span("Specialist RAG invocation"): + try: + # Get LLM with specified temperature + llm = self.model_variables.get_llm(temperature=self.temperature) + + # Get template + template = self.model_variables.get_template('rag') + language_template = create_language_template(template, language) + full_template = replace_variable_in_template( + language_template, + "{tenant_context}", + self.specialist_context + ) + + if self.tuning: + self._log_tuning("Template preparation completed", { + "template": full_template, + "context": formatted_context, + "tenant_context": self.specialist_context, + }) + + # Create prompt + rag_prompt = ChatPromptTemplate.from_template(full_template) + + # Setup chain components + setup_and_retrieval = RunnableParallel({ + "context": lambda x: formatted_context, + "question": lambda x: x + }) + + # Get output schema for structured output + output_schema = OutputRegistry.get_schema(self.type) + structured_llm = llm.with_structured_output(output_schema) + chain = setup_and_retrieval | rag_prompt | structured_llm + + raw_result = chain.invoke(query) + result = SpecialistResult.create_for_type( + "STANDARD_RAG", + detailed_query=detailed_question, + answer=raw_result.answer, + citations=[ctx.metadata.document_id for ctx in unique_contexts + if ctx.id in raw_result.citations], + insufficient_info=raw_result.insufficient_info + ) + + if self.tuning: + self._log_tuning("LLM chain execution completed", { + "Result": result.model_dump() + }) + + except Exception as e: + current_app.logger.error(f"Error in LLM processing: {e}") + if self.tuning: + self._log_tuning("LLM processing error", {"error": str(e)}) + raise + + return result + + except Exception as e: + current_app.logger.error(f'Error in RAG specialist execution: {str(e)}') + raise + + +# Register the specialist type +SpecialistRegistry.register("STANDARD_RAG", RAGSpecialist) +OutputRegistry.register("STANDARD_RAG", RAGOutput) diff --git a/eveai_chat_workers/specialists/registry.py b/eveai_chat_workers/specialists/registry.py new file mode 100644 index 0000000..b474b5b --- /dev/null +++ b/eveai_chat_workers/specialists/registry.py @@ -0,0 +1,21 @@ +from typing import Dict, Type +from .base import BaseSpecialist + + +class SpecialistRegistry: + """Registry for specialist types""" + + _registry: Dict[str, Type[BaseSpecialist]] = {} + + @classmethod + def register(cls, specialist_type: str, specialist_class: Type[BaseSpecialist]): + """Register a new specialist type""" + cls._registry[specialist_type] = specialist_class + + @classmethod + def get_specialist_class(cls, specialist_type: str) -> Type[BaseSpecialist]: + """Get the specialist class for a given type""" + if specialist_type not in cls._registry: + raise ValueError(f"Unknown specialist type: {specialist_type}") + return cls._registry[specialist_type] + diff --git a/eveai_chat_workers/specialists/specialist_typing.py b/eveai_chat_workers/specialists/specialist_typing.py new file mode 100644 index 0000000..7afb911 --- /dev/null +++ b/eveai_chat_workers/specialists/specialist_typing.py @@ -0,0 +1,144 @@ +from typing import List, Dict, Any, Optional +from pydantic import BaseModel, Field, model_validator +from config.specialist_types import SPECIALIST_TYPES +from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments + + +class SpecialistArguments(BaseModel): + """ + Dynamic arguments for specialists, allowing arbitrary fields but validating required ones + based on SPECIALIST_TYPES configuration. + """ + type: str = Field(..., description="Type of specialist (e.g. STANDARD_RAG)") + retriever_arguments: Dict[str, Any] = Field( + default_factory=dict, + description="Arguments for each retriever, keyed by retriever ID" + ) + + # Allow any additional fields + model_config = { + "extra": "allow" + } + + @model_validator(mode='after') + def validate_required_arguments(self) -> 'SpecialistArguments': + """Validate that all required arguments for this specialist type are present""" + specialist_config = SPECIALIST_TYPES.get(self.type) + if not specialist_config: + raise ValueError(f"Unknown specialist type: {self.type}") + + # Check required arguments from configuration + for arg_name, arg_config in specialist_config['arguments'].items(): + if arg_config.get('required', False): + if not hasattr(self, arg_name): + raise ValueError(f"Missing required argument '{arg_name}' for {self.type}") + + # Type validation + value = getattr(self, arg_name) + expected_type = arg_config['type'] + if expected_type == 'str' and not isinstance(value, str): + raise ValueError(f"Argument '{arg_name}' must be a string") + elif expected_type == 'int' and not isinstance(value, int): + raise ValueError(f"Argument '{arg_name}' must be an integer") + + return self + + @classmethod + def create(cls, type_name: str, specialist_args: Dict[str, Any], + retriever_args: Dict[str, Dict[str, Any]]) -> 'SpecialistArguments': + """ + Factory method to create SpecialistArguments with validated retriever arguments + + Args: + type_name: The specialist type (e.g., 'STANDARD_RAG') + specialist_args: Arguments specific to the specialist + retriever_args: Dictionary of retriever arguments keyed by retriever ID + + Returns: + Validated SpecialistArguments instance + """ + # Convert raw retriever arguments to RetrieverArguments instances + validated_retriever_args = {} + for retriever_id, args in retriever_args.items(): + # Ensure type is included in retriever arguments + if 'type' not in args: + raise ValueError(f"Retriever arguments for {retriever_id} must include 'type'") + + validated_retriever_args[retriever_id] = RetrieverArguments(**args) + + # Combine everything into the specialist arguments + return cls( + type=type_name, + **specialist_args, + retriever_arguments=validated_retriever_args + ) + + +class SpecialistResult(BaseModel): + """ + Dynamic results from specialists, validating required fields based on + SPECIALIST_TYPES configuration. + """ + type: str = Field(..., description="Type of specialist (e.g. STANDARD_RAG)") + + # Allow any additional fields + model_config = { + "extra": "allow" + } + + @model_validator(mode='after') + def validate_required_results(self) -> 'SpecialistResult': + """Validate that all required result fields for this specialist type are present""" + specialist_config = SPECIALIST_TYPES.get(self.type) + if not specialist_config: + raise ValueError(f"Unknown specialist type: {self.type}") + + # Check required results from configuration + required_results = specialist_config.get('results', {}) + for result_name, result_config in required_results.items(): + if result_config.get('required', False): + if not hasattr(self, result_name): + raise ValueError(f"Missing required result '{result_name}' for {self.type}") + + # Type validation + value = getattr(self, result_name) + expected_type = result_config['type'] + + # Validate based on type annotation + if expected_type == 'str' and not isinstance(value, str): + raise ValueError(f"Result '{result_name}' must be a string") + elif expected_type == 'bool' and not isinstance(value, bool): + raise ValueError(f"Result '{result_name}' must be a boolean") + elif expected_type == 'List[str]' and not ( + isinstance(value, list) and all(isinstance(x, str) for x in value)): + raise ValueError(f"Result '{result_name}' must be a list of strings") + # Add other type validations as needed + + return self + + @classmethod + def create_for_type(cls, specialist_type: str, **results) -> 'SpecialistResult': + """ + Factory method to create a type-specific result + + Args: + specialist_type: The type of specialist (e.g., 'STANDARD_RAG') + **results: The result values to include + + Returns: + Validated SpecialistResult instance + + Example: + For STANDARD_RAG: + result = SpecialistResult.create_for_type( + 'STANDARD_RAG', + answer="The answer text", + citations=["doc1", "doc2"], + insufficient_info=False + ) + """ + # Add the type to the results + results['type'] = specialist_type + + # Create and validate the result + return cls(**results) \ No newline at end of file diff --git a/eveai_chat_workers/tasks.py b/eveai_chat_workers/tasks.py index 6e124ed..eae77a0 100644 --- a/eveai_chat_workers/tasks.py +++ b/eveai_chat_workers/tasks.py @@ -1,24 +1,23 @@ from datetime import datetime as dt, timezone as tz +from typing import Dict, Any, Optional + from flask import current_app -from langchain_core.output_parsers import StrOutputParser -from langchain_core.runnables import RunnableParallel, RunnablePassthrough from sqlalchemy.exc import SQLAlchemyError -# OpenAI imports -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.exceptions import LangChainException - +from common.utils.config_field_types import TaggingFields from common.utils.database import Database -from common.models.document import Embedding +from common.models.document import Embedding, Catalog from common.models.user import Tenant -from common.models.interaction import ChatSession, Interaction, InteractionEmbedding -from common.extensions import db +from common.models.interaction import ChatSession, Interaction, InteractionEmbedding, Specialist, SpecialistRetriever +from common.extensions import db, cache_manager from common.utils.celery_utils import current_celery -from common.utils.model_utils import select_model_variables, create_language_template, replace_variable_in_template -from common.langchain.retrievers.eveai_default_rag_retriever import EveAIDefaultRagRetriever -from common.langchain.retrievers.eveai_history_retriever import EveAIHistoryRetriever from common.utils.business_event import BusinessEvent from common.utils.business_event_context import current_event +from config.specialist_types import SPECIALIST_TYPES +from eveai_chat_workers.chat_session_cache import get_chat_history +from eveai_chat_workers.specialists.registry import SpecialistRegistry +from config.retriever_types import RETRIEVER_TYPES +from eveai_chat_workers.specialists.specialist_typing import SpecialistArguments # Healthcheck task @@ -27,41 +26,207 @@ def ping(): return 'pong' -def detail_question(question, language, model_variables, session_id): - current_app.logger.debug(f'Detail question: {question}') - current_app.logger.debug(f'model_variables: {model_variables}') - current_app.logger.debug(f'session_id: {session_id}') - retriever = EveAIHistoryRetriever(model_variables=model_variables, session_id=session_id) - llm = model_variables['llm'] - template = model_variables['history_template'] - language_template = create_language_template(template, language) - full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context']) - history_prompt = ChatPromptTemplate.from_template(full_template) - setup_and_retrieval = RunnableParallel({"history": retriever, "question": RunnablePassthrough()}) - output_parser = StrOutputParser() - - chain = setup_and_retrieval | history_prompt | llm | output_parser - - try: - answer = chain.invoke(question) - return answer - except LangChainException as e: - current_app.logger.error(f'Error detailing question: {e}') - raise +class ArgumentPreparationError(Exception): + """Custom exception for argument preparation errors""" + pass -@current_celery.task(name='ask_question', queue='llm_interactions') -def ask_question(tenant_id, question, language, session_id, user_timezone, room): - """returns result structured as follows: - result = { - 'answer': 'Your answer here', - 'citations': ['http://example.com/citation1', 'http://example.com/citation2'], - 'algorithm': 'algorithm_name', - 'interaction_id': 'interaction_id_value' - } +def validate_specialist_arguments(specialist_type: str, arguments: Dict[str, Any]) -> None: """ - with BusinessEvent("Ask Question", tenant_id=tenant_id, chat_session_id=session_id): - current_app.logger.info(f'ask_question: Received question for tenant {tenant_id}: {question}. Processing...') + Validate specialist-specific arguments + + Args: + specialist_type: Type of specialist + arguments: Arguments to validate (excluding retriever-specific arguments) + + Raises: + ArgumentPreparationError: If validation fails + """ + specialist_config = SPECIALIST_TYPES.get(specialist_type) + if not specialist_config: + raise ArgumentPreparationError(f"Unknown specialist type: {specialist_type}") + + required_args = specialist_config.get('arguments', {}) + + # Check for required arguments + for arg_name, arg_config in required_args.items(): + if arg_config.get('required', False) and arg_name not in arguments: + raise ArgumentPreparationError(f"Missing required argument '{arg_name}' for specialist") + + if arg_name in arguments: + # Type checking + expected_type = arg_config.get('type') + if expected_type == 'str' and not isinstance(arguments[arg_name], str): + raise ArgumentPreparationError(f"Argument '{arg_name}' must be a string") + elif expected_type == 'int' and not isinstance(arguments[arg_name], int): + raise ArgumentPreparationError(f"Argument '{arg_name}' must be an integer") + + +def validate_retriever_arguments(retriever_type: str, arguments: Dict[str, Any], + catalog_config: Optional[Dict[str, Any]] = None) -> None: + """ + Validate retriever-specific arguments + + Args: + retriever_type: Type of retriever + arguments: Arguments to validate + catalog_config: Optional catalog configuration for metadata validation + + Raises: + ArgumentPreparationError: If validation fails + """ + retriever_config = RETRIEVER_TYPES.get(retriever_type) + if not retriever_config: + raise ArgumentPreparationError(f"Unknown retriever type: {retriever_type}") + + # Validate standard retriever arguments + required_args = retriever_config.get('arguments', {}) + for arg_name, arg_config in required_args.items(): + if arg_config.get('required', False) and arg_name not in arguments: + raise ArgumentPreparationError(f"Missing required argument '{arg_name}' for retriever") + + # Only validate metadata filters if catalog configuration is provided + if catalog_config and 'metadata_filters' in arguments: + if 'tagging_fields' in catalog_config: + tagging_fields = TaggingFields.from_dict(catalog_config['tagging_fields']) + errors = tagging_fields.validate_argument_values(arguments['metadata_filters']) + if errors: + raise ArgumentPreparationError(f"Invalid metadata filters: {errors}") + + +def is_retriever_id(key: str) -> bool: + """ + Check if a key represents a valid retriever ID. + Valid formats: positive integers, including leading zeros + + Args: + key: String to check + + Returns: + bool: True if the key represents a valid retriever ID + """ + try: + # Convert to int to handle leading zeros + value = int(key) + # Ensure it's a positive number + return value > 0 + except ValueError: + return False + + +def prepare_arguments(specialist: Any, arguments: Dict[str, Any]) -> Dict[str, Any]: + """ + Prepare complete argument dictionary for specialist execution with inheritance + + Args: + specialist: Specialist model instance + arguments: Dictionary containing: + - Specialist arguments + - Retriever-specific arguments keyed by retriever ID + + Returns: + Dict containing prepared arguments with inheritance applied + + Raises: + ArgumentPreparationError: If argument preparation or validation fails + """ + try: + # Separate specialist arguments from retriever arguments + retriever_args = {} + specialist_args = {} + + for key, value in arguments.items(): + if isinstance(key, str) and is_retriever_id(key): # Retriever ID + retriever_args[key] = value + else: + specialist_args[key] = value + + # Validate specialist arguments + validate_specialist_arguments(specialist.type, specialist_args) + + # Get all retrievers associated with this specialist + specialist_retrievers = ( + SpecialistRetriever.query + .filter_by(specialist_id=specialist.id) + .all() + ) + + # Process each retriever + prepared_retriever_args = {} + for spec_retriever in specialist_retrievers: + retriever = spec_retriever.retriever + retriever_id = str(retriever.id) + + # Get catalog configuration if it exists + catalog_config = None + if retriever.catalog_id: + try: + catalog = Catalog.query.get(retriever.catalog_id) + if catalog: + catalog_config = catalog.configuration + except SQLAlchemyError: + current_app.logger.warning( + f"Could not fetch catalog {retriever.catalog_id} for retriever {retriever_id}" + ) + + # Start with specialist arguments (inheritance) + inherited_args = specialist_args.copy() + + # Override with retriever-specific arguments if provided + if retriever_id in retriever_args: + inherited_args.update(retriever_args[retriever_id]) + + # Always include the retriever type + inherited_args['type'] = retriever.type + + # Validate the combined arguments + validate_retriever_arguments( + retriever.type, + inherited_args, + catalog_config + ) + + prepared_retriever_args[retriever_id] = inherited_args + + # Construct final argument structure + final_arguments = { + **specialist_args, + 'retriever_arguments': prepared_retriever_args + } + return final_arguments + + except SQLAlchemyError as e: + current_app.logger.error(f'Database error during argument preparation: {e}') + raise ArgumentPreparationError(f"Database error: {str(e)}") + except Exception as e: + current_app.logger.error(f'Error during argument preparation: {e}') + raise ArgumentPreparationError(str(e)) + + +@current_celery.task(name='execute_specialist', queue='llm_interactions') +def execute_specialist(tenant_id: int, specialist_id: int, arguments: Dict[str, Any], + session_id: str, user_timezone: str, room: str) -> dict: + """ + Execute a specialist with given arguments + + Args: + tenant_id: ID of the tenant + specialist_id: ID of the specialist to use + arguments: Dictionary containing all required arguments for specialist and retrievers + session_id: Chat session ID + user_timezone: User's timezone + room: Socket.IO room for the response + + Returns: + dict: { + 'result': Dict - Specialist execution result + 'interaction_id': int - Created interaction ID + 'room': str - Socket.IO room + } + """ + with BusinessEvent("Execute Specialist", tenant_id=tenant_id, chat_session_id=session_id) as event: + current_app.logger.info( + f'execute_specialist: Processing request for tenant {tenant_id} using specialist {specialist_id}') try: # Retrieve the tenant @@ -69,208 +234,85 @@ def ask_question(tenant_id, question, language, session_id, user_timezone, room) if not tenant: raise Exception(f'Tenant {tenant_id} not found.') - # Ensure we are working in the correct database schema + # Switch to correct database schema Database(tenant_id).switch_schema() - # Ensure we have a session to story history - chat_session = ChatSession.query.filter_by(session_id=session_id).first() - if not chat_session: + # Ensure we have a session + cached_session = cache_manager.chat_session_cache.get_cached_session( + session_id, + create_params={'timezone': user_timezone} + ) + + # Get specialist from database + specialist = Specialist.query.get_or_404(specialist_id) + + # Prepare complete arguments + try: + raw_arguments = prepare_arguments(specialist, arguments) + # Convert the prepared arguments into a SpecialistArguments instance + complete_arguments = SpecialistArguments.create( + type_name=specialist.type, + specialist_args={k: v for k, v in raw_arguments.items() if k != 'retriever_arguments'}, + retriever_args=raw_arguments.get('retriever_arguments', {}) + ) + except ValueError as e: + current_app.logger.error(f'execute_specialist: Error preparing arguments: {e}') + raise + + # Create new interaction record + new_interaction = Interaction() + new_interaction.chat_session_id = cached_session.id + new_interaction.timezone = user_timezone + new_interaction.question_at = dt.now(tz.utc) + new_interaction.specialist_id = specialist.id + new_interaction.specialist_arguments = complete_arguments.model_dump(mode='json') + + try: + db.session.add(new_interaction) + db.session.commit() + event.update_attribute('interaction_id', new_interaction.id) + + except SQLAlchemyError as e: + current_app.logger.error(f'execute_specialist: Error creating interaction: {e}') + raise + + with current_event.create_span("Specialist invocation"): + # Initialize specialist instance + specialist_class = SpecialistRegistry.get_specialist_class(specialist.type) + specialist_instance = specialist_class( + tenant_id=tenant_id, + specialist_id=specialist_id, + session_id=session_id, + ) + + # Execute specialist + result = specialist_instance.execute(complete_arguments) + + # Update interaction record + new_interaction.specialist_results = result.model_dump(mode='json') # Store complete result + new_interaction.answer_at = dt.now(tz.utc) + try: - chat_session = ChatSession() - chat_session.session_id = session_id - chat_session.session_start = dt.now(tz.utc) - chat_session.timezone = user_timezone - db.session.add(chat_session) + db.session.add(new_interaction) db.session.commit() except SQLAlchemyError as e: - current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}') + current_app.logger.error(f'execute_specialist: Error updating interaction: {e}') raise - with current_event.create_span("RAG Answer"): - result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session) - result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name'] - result['interaction_id'] = interaction.id - result['room'] = room # Include the room in the result + # Now that we have a complete interaction with an answer, add it to the cache + cache_manager.chat_session_cache.add_completed_interaction(session_id, new_interaction) - if result['insufficient_info']: - if 'LLM' in tenant.fallback_algorithms: - with current_event.create_span("Fallback Algorithm LLM"): - result, interaction = answer_using_llm(question, language, tenant, chat_session) - result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name'] - result['interaction_id'] = interaction.id - result['room'] = room # Include the room in the result - - return result - except Exception as e: - current_app.logger.error(f'ask_question: Error processing question: {e}') - raise - - -def answer_using_tenant_rag(question, language, tenant, chat_session): - new_interaction = Interaction() - new_interaction.question = question - new_interaction.language = language - new_interaction.timezone = chat_session.timezone - new_interaction.appreciation = None - new_interaction.chat_session_id = chat_session.id - new_interaction.question_at = dt.now(tz.utc) - new_interaction.algorithm_used = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name'] - - # Select variables to work with depending on tenant model - model_variables = select_model_variables(tenant) - tenant_info = tenant.to_dict() - - # Langchain debugging if required - # set_debug(True) - - with current_event.create_span("Detail Question"): - detailed_question = detail_question(question, language, model_variables, chat_session.session_id) - if model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.') - current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------') - new_interaction.detailed_question = detailed_question - new_interaction.detailed_question_at = dt.now(tz.utc) - - with current_event.create_span("Generate Answer using RAG"): - retriever = EveAIDefaultRagRetriever(1, model_variables, tenant_info) - llm = model_variables['llm'] - template = model_variables['rag_template'] - language_template = create_language_template(template, language) - full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context']) - rag_prompt = ChatPromptTemplate.from_template(full_template) - setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()}) - if model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.') - current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------') - - new_interaction_embeddings = [] - if not model_variables['cited_answer_cls']: # The model doesn't support structured feedback - output_parser = StrOutputParser() - - chain = setup_and_retrieval | rag_prompt | llm | output_parser - - # Invoke the chain with the actual question - answer = chain.invoke(detailed_question) - new_interaction.answer = answer - result = { - 'answer': answer, - 'citations': [], - 'insufficient_info': False + # Prepare response + response = { + 'result': result.model_dump(), + 'interaction_id': new_interaction.id, + 'room': room } - else: # The model supports structured feedback - structured_llm = llm.with_structured_output(model_variables['cited_answer_cls']) + return response - chain = setup_and_retrieval | rag_prompt | structured_llm - - result = chain.invoke(detailed_question).dict() - current_app.logger.debug(f'ask_question: result answer: {result['answer']}') - current_app.logger.debug(f'ask_question: result citations: {result["citations"]}') - current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}') - if model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}') - current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}') - current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}') - current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------') - new_interaction.answer = result['answer'] - - # Filter out the existing Embedding IDs - given_embedding_ids = [int(emb_id) for emb_id in result['citations']] - embeddings = ( - db.session.query(Embedding) - .filter(Embedding.id.in_(given_embedding_ids)) - .all() - ) - existing_embedding_ids = [emb.id for emb in embeddings] - urls = list(set(emb.document_version.url for emb in embeddings)) - if model_variables['rag_tuning']: - current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n') - current_app.rag_tuning_logger.debug(f'{urls}') - current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------') - - for emb_id in existing_embedding_ids: - new_interaction_embedding = InteractionEmbedding(embedding_id=emb_id) - new_interaction_embedding.interaction = new_interaction - new_interaction_embeddings.append(new_interaction_embedding) - - result['citations'] = urls - - # Disable langchain debugging if set above. - # set_debug(False) - - new_interaction.answer_at = dt.now(tz.utc) - chat_session.session_end = dt.now(tz.utc) - - try: - db.session.add(chat_session) - db.session.add(new_interaction) - db.session.add_all(new_interaction_embeddings) - db.session.commit() - return result, new_interaction - except SQLAlchemyError as e: - current_app.logger.error(f'ask_question: Error saving interaction to database: {e}') - raise - - -def answer_using_llm(question, language, tenant, chat_session): - new_interaction = Interaction() - new_interaction.question = question - new_interaction.language = language - new_interaction.timezone = chat_session.timezone - new_interaction.appreciation = None - new_interaction.chat_session_id = chat_session.id - new_interaction.question_at = dt.now(tz.utc) - new_interaction.algorithm_used = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name'] - - # Select variables to work with depending on tenant model - model_variables = select_model_variables(tenant) - tenant_info = tenant.to_dict() - - # Langchain debugging if required - # set_debug(True) - - with current_event.create_span("Detail Question"): - detailed_question = detail_question(question, language, model_variables, chat_session.session_id) - current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}') - new_interaction.detailed_question = detailed_question - new_interaction.detailed_question_at = dt.now(tz.utc) - - with current_event.create_span("Detail Answer using LLM"): - retriever = EveAIDefaultRagRetriever(1, model_variables, tenant_info) - llm = model_variables['llm_no_rag'] - template = model_variables['encyclopedia_template'] - language_template = create_language_template(template, language) - rag_prompt = ChatPromptTemplate.from_template(language_template) - setup = RunnablePassthrough() - output_parser = StrOutputParser() - - new_interaction_embeddings = [] - - chain = setup | rag_prompt | llm | output_parser - input_question = {"question": detailed_question} - - # Invoke the chain with the actual question - answer = chain.invoke(input_question) - new_interaction.answer = answer - result = { - 'answer': answer, - 'citations': [], - 'insufficient_info': False - } - - # Disable langchain debugging if set above. - # set_debug(False) - - new_interaction.answer_at = dt.now(tz.utc) - chat_session.session_end = dt.now(tz.utc) - - try: - db.session.add(chat_session) - db.session.add(new_interaction) - db.session.commit() - return result, new_interaction - except SQLAlchemyError as e: - current_app.logger.error(f'ask_question: Error saving interaction to database: {e}') + except Exception as e: + current_app.logger.error(f'execute_specialist: Error executing specialist: {e}') raise diff --git a/eveai_entitlements/tasks.py b/eveai_entitlements/tasks.py index 67de8b7..91bbf20 100644 --- a/eveai_entitlements/tasks.py +++ b/eveai_entitlements/tasks.py @@ -45,20 +45,14 @@ def update_usages(): max_timestamp = max(log.timestamp for log in logs) # Retrieve relevant LicenseUsage records - current_app.logger.debug(f"Searching relevant usages for tenant {tenant_id}") license_usages = get_relevant_license_usages(db.session, tenant_id, min_timestamp, max_timestamp) - current_app.logger.debug(f"Found {license_usages}, end searching relevant usages for tenant {tenant_id}") # Split logs based on LicenseUsage periods - current_app.logger.debug(f"Splitting usages for tenant {tenant_id}") logs_by_usage = split_logs_by_license_usage(logs, license_usages) - current_app.logger.debug(f"Found {logs_by_usage}, end splitting logs for tenant {tenant_id}") # Now you can process logs for each LicenseUsage for license_usage_id, logs in logs_by_usage.items(): - current_app.logger.debug(f"Processing logs for usage id {license_usage_id} for tenant {tenant_id}") process_logs_for_license_usage(tenant_id, license_usage_id, logs) - current_app.logger.debug(f"Finished processing logs for tenant {tenant_id}") except Exception as e: error = f"Usage Calculation error for Tenant {tenant_id}: {e}" error_list.append(error) @@ -212,8 +206,6 @@ def process_logs_for_license_usage(tenant_id, license_usage_id, logs): license_usage.interaction_completion_tokens_used += interaction_completion_tokens_used license_usage.interaction_total_tokens_used += interaction_total_tokens_used - current_app.logger.debug(f"Processed logs for license usage {license_usage.id}:\n{license_usage}") - # Commit the updates to the LicenseUsage and log records try: db.session.add(license_usage) @@ -232,7 +224,6 @@ def recalculate_storage_for_tenant(tenant): SELECT SUM(file_size) FROM document_version """)).scalar() - current_app.logger.debug(f"Recalculating storage for tenant {tenant} - Total storage: {total_storage}") # Update the LicenseUsage with the recalculated storage license_usage = db.session.query(LicenseUsage).filter_by(tenant_id=tenant.id).first() diff --git a/eveai_workers/__init__.py b/eveai_workers/__init__.py index 73efa2c..3ba5d53 100644 --- a/eveai_workers/__init__.py +++ b/eveai_workers/__init__.py @@ -4,10 +4,12 @@ from flask import Flask import os from common.utils.celery_utils import make_celery, init_celery -from common.extensions import db, minio_client -from config.logging_config import LOGGING +from common.extensions import db, minio_client, template_manager, cache_manager +import config.logging_config as logging_config from config.config import get_config +from . import processors + def create_app(config_file=None): app = Flask(__name__) @@ -22,8 +24,7 @@ def create_app(config_file=None): case _: app.config.from_object(get_config('dev')) - logging.config.dictConfig(LOGGING) - app.embed_tuning_logger = logging.getLogger('embed_tuning') + logging.config.dictConfig(logging_config.LOGGING) register_extensions(app) @@ -41,6 +42,8 @@ def create_app(config_file=None): def register_extensions(app): db.init_app(app) minio_client.init_app(app) + cache_manager.init_app(app) + template_manager.init_app(app) app, celery = create_app() diff --git a/eveai_workers/processors/__init__.py b/eveai_workers/processors/__init__.py new file mode 100644 index 0000000..380b4b9 --- /dev/null +++ b/eveai_workers/processors/__init__.py @@ -0,0 +1,5 @@ +# Import all processor implementations to ensure registration +from . import audio_processor, html_processor, pdf_processor + +# List of all available processor implementations +__all__ = ['audio_processor', 'html_processor', 'pdf_processor'] \ No newline at end of file diff --git a/eveai_workers/Processors/audio_processor.py b/eveai_workers/processors/audio_processor.py similarity index 74% rename from eveai_workers/Processors/audio_processor.py rename to eveai_workers/processors/audio_processor.py index 7ef3bd2..30518ee 100644 --- a/eveai_workers/Processors/audio_processor.py +++ b/eveai_workers/processors/audio_processor.py @@ -8,20 +8,20 @@ import tempfile from common.extensions import minio_client import subprocess -from .transcription_processor import TranscriptionProcessor +from .processor_registry import ProcessorRegistry +from .transcription_processor import TranscriptionBaseProcessor from common.utils.business_event_context import current_event -class AudioProcessor(TranscriptionProcessor): - def __init__(self, tenant, model_variables, document_version): - super().__init__(tenant, model_variables, document_version) - self.transcription_client = model_variables['transcription_client'] - self.transcription_model = model_variables['transcription_model'] +class AudioProcessor(TranscriptionBaseProcessor): + def __init__(self, tenant, model_variables, document_version, catalog, processor): + super().__init__(tenant, model_variables, document_version, catalog, processor) + self.transcription_model = model_variables.transcription_model self.ffmpeg_path = 'ffmpeg' - self.max_compression_duration = model_variables['max_compression_duration'] - self.max_transcription_duration = model_variables['max_transcription_duration'] - self.compression_cpu_limit = model_variables.get('compression_cpu_limit', 50) # CPU usage limit in percentage - self.compression_process_delay = model_variables.get('compression_process_delay', 0.1) # Delay between processing chunks in seconds + self.max_compression_duration = model_variables.max_compression_duration + self.max_transcription_duration = model_variables.max_transcription_duration + self.compression_cpu_limit = model_variables.compression_cpu_limit # CPU usage limit in percentage + self.compression_process_delay = model_variables.compression_process_delay # Delay between processing chunks in seconds self.file_type = document_version.file_type def _get_transcription(self): @@ -39,26 +39,25 @@ class AudioProcessor(TranscriptionProcessor): return transcription def _compress_audio(self, audio_data): - self._log("Compressing audio") - with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{self.document_version.file_type}') as temp_file: temp_file.write(audio_data) temp_file_path = temp_file.name try: - self._log("Creating AudioSegment from file") audio_info = AudioSegment.from_file(temp_file_path, format=self.document_version.file_type) - self._log("Finished creating AudioSegment from file") total_duration = len(audio_info) - self._log(f"Audio duration: {total_duration / 1000} seconds") - + self._log_tuning("_compress_audio", { + "Audio Duration (ms)": total_duration, + }) segment_length = self.max_compression_duration * 1000 # Convert to milliseconds total_chunks = (total_duration + segment_length - 1) // segment_length compressed_segments = AudioSegment.empty() for i in range(total_chunks): - self._log(f"Compressing segment {i + 1} of {total_chunks}") + self._log_tuning("_compress_audio", { + "Segment Nr": f"{i + 1} of {total_chunks}" + }) start_time = i * segment_length end_time = min((i + 1) * segment_length, total_duration) @@ -88,7 +87,9 @@ class AudioProcessor(TranscriptionProcessor): compressed_filename, compressed_buffer.read() ) - self._log(f"Saved compressed audio to MinIO: {compressed_filename}") + self._log_tuning("_compress_audio", { + "Compressed audio to MinIO": compressed_filename + }) return compressed_segments @@ -131,7 +132,6 @@ class AudioProcessor(TranscriptionProcessor): return compressed_segment def _transcribe_audio(self, audio_data): - self._log("Starting audio transcription") # audio = AudioSegment.from_file(io.BytesIO(audio_data), format="mp3") audio = audio_data @@ -140,7 +140,6 @@ class AudioProcessor(TranscriptionProcessor): total_chunks = len(audio) // segment_length + 1 for i, chunk in enumerate(audio[::segment_length]): - self._log(f'Processing chunk {i + 1} of {total_chunks}') segment_duration = 0 if i == total_chunks - 1: segment_duration = (len(audio) % segment_length) // 1000 @@ -153,37 +152,34 @@ class AudioProcessor(TranscriptionProcessor): try: file_size = os.path.getsize(temp_audio.name) - self._log(f"Temporary audio file size: {file_size} bytes") with open(temp_audio.name, 'rb') as audio_file: - file_start = audio_file.read(100) - self._log(f"First 100 bytes of audio file: {file_start}") - audio_file.seek(0) # Reset file pointer to the beginning - - self._log("Calling transcription API") - transcription = self.model_variables.transcribe( + transcription = self.model_variables.transcription_model.transcribe( file=audio_file, - model=self.transcription_model, language=self.document_version.language, response_format='verbose_json', - duration=segment_duration, + duration=segment_duration ) - self._log("Transcription API call completed") - if transcription: + trans = "" # Handle the transcription result based on its type if isinstance(transcription, str): - self._log(f"Transcription result (string): {transcription[:100]}...") - transcriptions.append(transcription) + trans = transcription elif hasattr(transcription, 'text'): - self._log( - f"Transcription result (object with 'text' attribute): {transcription.text[:100]}...") - transcriptions.append(transcription.text) + trans = transcription.text else: - self._log(f"Transcription result (unknown type): {str(transcription)[:100]}...") transcriptions.append(str(transcription)) + + transcriptions.append(trans) + + self._log_tuning("_transcribe_audio", { + "Chunk Nr": f"{i + 1} of {total_chunks}", + "Segment Duration": segment_duration, + "Transcription": trans, + }) else: self._log("Warning: Received empty transcription", level='warning') + self._log_tuning("_transcribe_audio", {"ERROR": "No transcription"}) except Exception as e: self._log(f"Error during transcription: {str(e)}", level='error') @@ -206,7 +202,10 @@ class AudioProcessor(TranscriptionProcessor): transcription_filename, full_transcription.encode('utf-8') ) - self._log(f"Saved transcription to MinIO: {transcription_filename}") + self._log_tuning(f"Saved transcription to MinIO: {transcription_filename}") return full_transcription + +# Register the processor +ProcessorRegistry.register("AUDIO_PROCESSOR", AudioProcessor) diff --git a/eveai_workers/Processors/processor.py b/eveai_workers/processors/base_processor.py similarity index 50% rename from eveai_workers/Processors/processor.py rename to eveai_workers/processors/base_processor.py index 361777a..5cf6358 100644 --- a/eveai_workers/Processors/processor.py +++ b/eveai_workers/processors/base_processor.py @@ -1,14 +1,42 @@ from abc import ABC, abstractmethod +from typing import Dict, Any + from flask import current_app from common.extensions import minio_client +from config.logging_config import TuningLogger -class Processor(ABC): - def __init__(self, tenant, model_variables, document_version): +class BaseProcessor(ABC): + def __init__(self, tenant, model_variables, document_version, catalog, processor): self.tenant = tenant self.model_variables = model_variables self.document_version = document_version - self.embed_tuning = model_variables['embed_tuning'] + self.catalog = catalog + self.processor = processor + self.tuning = processor.tuning if processor else False + self.tuning_logger = None + self._setup_tuning_logger() + + self._log_tuning("Processor initialized", { + "processor_type": processor.type if processor else None, + "document_version": document_version.id if document_version else None, + "catalog": catalog.id if catalog else None + }) + + def _setup_tuning_logger(self): + try: + self.tuning_logger = TuningLogger( + 'tuning', + tenant_id=self.tenant.id if self.tenant else None, + catalog_id=self.catalog.id if self.catalog else None, + processor_id=self.processor.id if self.processor else None, + ) + # Verify logger is working with a test message + if self.tuning: + self.tuning_logger.log_tuning('processor', "Tuning logger initialized") + except Exception as e: + current_app.logger.error(f"Failed to setup tuning logger: {str(e)}") + raise @abstractmethod def process(self): @@ -50,3 +78,11 @@ class Processor(ABC): return markdown + def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None: + if self.tuning and self.tuning_logger: + try: + self.tuning_logger.log_tuning('processor', message, data) + except Exception as e: + current_app.logger.error(f"Processor: Error in tuning logging: {e}") + + diff --git a/eveai_workers/Processors/html_processor.py b/eveai_workers/processors/html_processor.py similarity index 76% rename from eveai_workers/Processors/html_processor.py rename to eveai_workers/processors/html_processor.py index 9f0ddb4..e47553f 100644 --- a/eveai_workers/Processors/html_processor.py +++ b/eveai_workers/processors/html_processor.py @@ -4,21 +4,34 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough from common.extensions import db, minio_client from common.utils.model_utils import create_language_template -from .processor import Processor +from .base_processor import BaseProcessor from common.utils.business_event_context import current_event +from .processor_registry import ProcessorRegistry +from common.utils.string_list_converter import StringListConverter as SLC -class HTMLProcessor(Processor): - def __init__(self, tenant, model_variables, document_version): - super().__init__(tenant, model_variables, document_version) - self.html_tags = model_variables['html_tags'] - self.html_end_tags = model_variables['html_end_tags'] - self.html_included_elements = model_variables['html_included_elements'] - self.html_excluded_elements = model_variables['html_excluded_elements'] - self.html_excluded_classes = model_variables['html_excluded_classes'] - self.chunk_size = model_variables['processing_chunk_size'] # Adjust this based on your LLM's optimal input size - self.chunk_overlap = model_variables[ - 'processing_chunk_overlap'] # Adjust for context preservation between chunks +class HTMLProcessor(BaseProcessor): + def __init__(self, tenant, model_variables, document_version, catalog, processor): + super().__init__(tenant, model_variables, document_version, catalog, processor) + cat_conf = catalog.configuration + proc_conf = processor.configuration + self.html_tags = SLC.string_to_list(proc_conf['html_tags']) + self.html_end_tags = SLC.string_to_list(proc_conf['html_end_tags']) + self.html_included_elements = SLC.string_to_list(proc_conf['html_included_elements']) + self.html_excluded_elements = SLC.string_to_list(proc_conf['html_excluded_elements']) + self.html_excluded_classes = SLC.string_to_list(proc_conf['html_excluded_classes']) + self.tuning = self.processor.tuning + # Add verification logging + self._log(f"HTML Processor initialized with tuning={self.tuning}") + if self.tuning: + self._log_tuning("HTML Processor initialized", { + "html_tags": self.html_tags, + "html_end_tags": self.html_end_tags, + "included_elements": self.html_included_elements, + "excluded_elements": self.html_excluded_elements + }) + + self.chunk_size = catalog.max_chunk_size def process(self): self._log("Starting HTML processing") @@ -62,13 +75,14 @@ class HTMLProcessor(Processor): title = soup.find('title').get_text(strip=True) if soup.find('title') else '' self._log(f'Finished parsing HTML for tenant {self.tenant.id}') + self._log_tuning("_parse_html", {"extracted_html": extracted_html, "title": title}) return extracted_html, title def _generate_markdown_from_html(self, html_content): self._log(f'Generating markdown from HTML for tenant {self.tenant.id}') - llm = self.model_variables['llm'] - template = self.model_variables['html_parse_template'] + llm = self.model_variables.get_llm() + template = self.model_variables.get_template("html_parse") parse_prompt = ChatPromptTemplate.from_template(template) setup = RunnablePassthrough() output_parser = StrOutputParser() @@ -79,13 +93,10 @@ class HTMLProcessor(Processor): markdown_chunks = [] for chunk in chunks: - if self.embed_tuning: - self._log(f'Processing chunk: \n{chunk}\n') input_html = {"html": chunk} markdown_chunk = chain.invoke(input_html) markdown_chunks.append(markdown_chunk) - if self.embed_tuning: - self._log(f'Processed markdown chunk: \n{markdown_chunk}\n') + self._log_tuning("_generate_markdown_from_html", {"chunk": chunk, "markdown_chunk": markdown_chunk}) markdown = "\n\n".join(markdown_chunks) self._log(f'Finished generating markdown from HTML for tenant {self.tenant.id}') @@ -146,3 +157,7 @@ class HTMLProcessor(Processor): def _extract_element_content(self, element): content = ' '.join(child.strip() for child in element.stripped_strings) return f'<{element.name}>{content}\n' + + +# Register the processor +ProcessorRegistry.register("HTML_PROCESSOR", HTMLProcessor) \ No newline at end of file diff --git a/eveai_workers/Processors/pdf_processor.py b/eveai_workers/processors/pdf_processor.py similarity index 87% rename from eveai_workers/Processors/pdf_processor.py rename to eveai_workers/processors/pdf_processor.py index afa772e..91e4b2d 100644 --- a/eveai_workers/Processors/pdf_processor.py +++ b/eveai_workers/processors/pdf_processor.py @@ -9,18 +9,18 @@ from langchain_core.runnables import RunnablePassthrough from common.extensions import minio_client from common.utils.model_utils import create_language_template -from .processor import Processor +from .base_processor import BaseProcessor from common.utils.business_event_context import current_event +from .processor_registry import ProcessorRegistry -class PDFProcessor(Processor): - def __init__(self, tenant, model_variables, document_version): - super().__init__(tenant, model_variables, document_version) - # PDF-specific initialization - self.chunk_size = model_variables['processing_chunk_size'] - self.chunk_overlap = model_variables['processing_chunk_overlap'] - self.min_chunk_size = model_variables['processing_min_chunk_size'] - self.max_chunk_size = model_variables['processing_max_chunk_size'] +class PDFProcessor(BaseProcessor): + def __init__(self, tenant, model_variables, document_version, catalog, processor): + super().__init__(tenant, model_variables, document_version, catalog, processor) + + self.chunk_size = catalog.max_chunk_size + self.chunk_overlap = 0 + self.tuning = self.processor.tuning def process(self): self._log("Starting PDF processing") @@ -38,7 +38,8 @@ class PDFProcessor(Processor): with current_event.create_span("Markdown Generation"): llm_chunks = self._split_content_for_llm(structured_content) markdown = self._process_chunks_with_llm(llm_chunks) - self._save_markdown(markdown) + + self._save_markdown(markdown) self._log("Finished processing PDF") return markdown, title except Exception as e: @@ -56,19 +57,10 @@ class PDFProcessor(Processor): 'figures': self._extract_figures(page, page_num, figure_counter), 'tables': self._extract_tables(page) } - if self.embed_tuning: - self._log(f'Extracted PDF Content for page {page_num + 1}') - self._log(f"{page_content }") + self._log_tuning("_extract_content", {"page_num": page_num, "page_content": page_content}) figure_counter += len(page_content['figures']) extracted_content.append(page_content) - # if self.embed_tuning: - # current_app.embed_tuning_logger.debug(f'Extracted PDF Content') - # current_app.embed_tuning_logger.debug(f'---------------------') - # current_app.embed_tuning_logger.debug(f'Page: {page_content}') - # current_app.embed_tuning_logger.debug(f'End of Extracted PDF Content') - # current_app.embed_tuning_logger.debug(f'----------------------------') - return extracted_content def _extract_figures(self, page, page_num, figure_counter): @@ -127,6 +119,7 @@ class PDFProcessor(Processor): markdown_table = self._table_to_markdown(table) if markdown_table: # Only add non-empty tables tables.append(markdown_table) + self._log_tuning("_extract_tables", {"markdown_table": markdown_table}) except Exception as e: self._log(f"Error extracting tables from page: {str(e)}", level='error') return tables @@ -202,7 +195,7 @@ class PDFProcessor(Processor): for table in page['tables']: structured_content += f"\n{table}\n" - if self.embed_tuning: + if self.tuning: self._save_intermediate(structured_content, "structured_content.md") return structured_content, title @@ -217,8 +210,8 @@ class PDFProcessor(Processor): return text_splitter.split_text(content) def _process_chunks_with_llm(self, chunks): - llm = self.model_variables['llm'] - template = self.model_variables['pdf_parse_template'] + llm = self.model_variables.get_llm() + template = self.model_variables.get_template('pdf_parse') pdf_prompt = ChatPromptTemplate.from_template(template) setup = RunnablePassthrough() output_parser = StrOutputParser() @@ -232,3 +225,7 @@ class PDFProcessor(Processor): markdown_chunks.append(result) return "\n\n".join(markdown_chunks) + + +# Register the processor +ProcessorRegistry.register("PDF_PROCESSOR", PDFProcessor) diff --git a/eveai_workers/processors/processor_registry.py b/eveai_workers/processors/processor_registry.py new file mode 100644 index 0000000..6390016 --- /dev/null +++ b/eveai_workers/processors/processor_registry.py @@ -0,0 +1,92 @@ +from typing import Dict, Type, Optional +from flask import current_app +from config.processor_types import PROCESSOR_TYPES +from .base_processor import BaseProcessor + + +class ProcessorRegistry: + """Registry for processor types that aligns with PROCESSOR_TYPES configuration""" + + _registry: Dict[str, Type[BaseProcessor]] = {} + + @classmethod + def register(cls, processor_type: str, processor_class: Type[BaseProcessor]): + """ + Register a new processor type that must match a type in PROCESSOR_TYPES + + Args: + processor_type: Type identifier from PROCESSOR_TYPES + processor_class: Processor implementation class + + Raises: + ValueError: If processor_type isn't defined in PROCESSOR_TYPES + """ + if processor_type not in PROCESSOR_TYPES: + raise ValueError(f"Processor type {processor_type} not found in PROCESSOR_TYPES configuration") + + cls._registry[processor_type] = processor_class + + @classmethod + def get_processor_class(cls, processor_type: str) -> Type[BaseProcessor]: + """ + Get the processor class for a given processor type + + Args: + processor_type: Type identifier from PROCESSOR_TYPES + + Returns: + The registered processor class + + Raises: + ValueError: If no processor is registered for the given type + """ + if processor_type not in cls._registry: + raise ValueError(f"No processor registered for type: {processor_type}") + return cls._registry[processor_type] + + @classmethod + def get_processor_for_file_type(cls, file_type: str) -> tuple[str, Type[BaseProcessor]]: + """ + Find appropriate processor for a file type by checking PROCESSOR_TYPES definitions + + Args: + file_type: File extension (e.g., 'html', 'pdf') + + Returns: + Tuple of (processor_type, processor_class) + + Raises: + ValueError: If no processor is found for the file type + """ + # First find which processor type handles this file type + for proc_type, config in PROCESSOR_TYPES.items(): + # Check if file_type is in the supported file_types (handling both string and list formats) + supported_types = config['file_types'] + if isinstance(supported_types, str): + supported_types = [t.strip() for t in supported_types.split(',')] + + if file_type in supported_types: + # Get the registered processor class for this type + if proc_type in cls._registry: + return proc_type, cls._registry[proc_type] + else: + raise ValueError( + f"Found processor type {proc_type} for file type {file_type} but no processor is registered") + + raise ValueError(f"No processor type found for file type: {file_type}") + + @classmethod + def validate_processor_registration(cls): + """ + Validate that all PROCESSOR_TYPES have registered processors + + Raises: + ValueError: If any processor type lacks a registered processor + """ + missing_processors = [] + for proc_type in PROCESSOR_TYPES.keys(): + if proc_type not in cls._registry: + missing_processors.append(proc_type) + + if missing_processors: + raise ValueError(f"Missing processor registrations for: {', '.join(missing_processors)}") diff --git a/eveai_workers/Processors/srt_processor.py b/eveai_workers/processors/srt_processor.py similarity index 90% rename from eveai_workers/Processors/srt_processor.py rename to eveai_workers/processors/srt_processor.py index eb7c8fc..ae3526e 100644 --- a/eveai_workers/Processors/srt_processor.py +++ b/eveai_workers/processors/srt_processor.py @@ -1,9 +1,9 @@ from common.extensions import minio_client -from .transcription_processor import TranscriptionProcessor +from .transcription_processor import TranscriptionBaseProcessor import re -class SRTProcessor(TranscriptionProcessor): +class SRTProcessor(TranscriptionBaseProcessor): def _get_transcription(self): file_data = minio_client.download_document_file( self.tenant.id, diff --git a/eveai_workers/Processors/transcription_processor.py b/eveai_workers/processors/transcription_processor.py similarity index 80% rename from eveai_workers/Processors/transcription_processor.py rename to eveai_workers/processors/transcription_processor.py index 837c7a5..8db2c82 100644 --- a/eveai_workers/Processors/transcription_processor.py +++ b/eveai_workers/processors/transcription_processor.py @@ -5,15 +5,15 @@ from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough from common.utils.model_utils import create_language_template -from .processor import Processor +from .base_processor import BaseProcessor from common.utils.business_event_context import current_event -class TranscriptionProcessor(Processor): - def __init__(self, tenant, model_variables, document_version): - super().__init__(tenant, model_variables, document_version) - self.chunk_size = model_variables['processing_chunk_size'] - self.chunk_overlap = model_variables['processing_chunk_overlap'] +class TranscriptionBaseProcessor(BaseProcessor): + def __init__(self, tenant, model_variables, document_version, catalog, processor): + super().__init__(tenant, model_variables, document_version, catalog, processor) + self.annotation_chunk_size = model_variables.annotation_chunk_length + self.annotation_chunk_overlap = 0 def process(self): self._log("Starting Transcription processing") @@ -37,17 +37,17 @@ class TranscriptionProcessor(Processor): def _chunk_transcription(self, transcription): text_splitter = RecursiveCharacterTextSplitter( - chunk_size=self.chunk_size, - chunk_overlap=self.chunk_overlap, + chunk_size=self.annotation_chunk_size, + chunk_overlap=self.annotation_chunk_overlap, length_function=len, separators=["\n\n", "\n", " ", ""] ) return text_splitter.split_text(transcription) def _process_chunks(self, chunks): - self._log("Generating markdown from transcription") - llm = self.model_variables['llm'] - template = self.model_variables['transcript_template'] + self._log_tuning("_process_chunks", {"Nr of Chunks": len(chunks)}) + llm = self.model_variables.get_llm() + template = self.model_variables.get_template('transcript') language_template = create_language_template(template, self.document_version.language) transcript_prompt = ChatPromptTemplate.from_template(language_template) setup = RunnablePassthrough() @@ -58,14 +58,18 @@ class TranscriptionProcessor(Processor): markdown_chunks = [] previous_part = "" for i, chunk in enumerate(chunks): - self._log(f"Processing chunk {i + 1} of {len(chunks)}") - self._log(f"Previous part: {previous_part}") input_transcript = { 'transcript': chunk, 'previous_part': previous_part } markdown = chain.invoke(input_transcript) markdown = self._clean_markdown(markdown) + self._log_tuning("_process_chunks", { + "Chunk Number": f"{i + 1} of {len(chunks)}", + "Chunk": chunk, + "Previous Chunk": previous_part, + "Markdown": markdown, + }) markdown_chunks.append(markdown) # Extract the last part for the next iteration diff --git a/eveai_workers/tasks.py b/eveai_workers/tasks.py index 50ad4be..12a2e58 100644 --- a/eveai_workers/tasks.py +++ b/eveai_workers/tasks.py @@ -10,22 +10,20 @@ from langchain_core.exceptions import LangChainException from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough +from sqlalchemy import or_ from sqlalchemy.exc import SQLAlchemyError -from common.extensions import db, minio_client -from common.models.document import DocumentVersion, Embedding, Document +from common.extensions import db, minio_client, template_manager +from common.models.document import DocumentVersion, Embedding, Document, Processor, Catalog from common.models.user import Tenant from common.utils.celery_utils import current_celery from common.utils.database import Database -from common.utils.model_utils import select_model_variables, create_language_template -from common.utils.os_utils import safe_remove, sync_folder -from eveai_workers.Processors.audio_processor import AudioProcessor -from eveai_workers.Processors.html_processor import HTMLProcessor -from eveai_workers.Processors.pdf_processor import PDFProcessor -from eveai_workers.Processors.srt_processor import SRTProcessor +from common.utils.model_utils import create_language_template, get_model_variables from common.utils.business_event import BusinessEvent from common.utils.business_event_context import current_event +from config.processor_types import PROCESSOR_TYPES +from eveai_workers.processors.processor_registry import ProcessorRegistry # Healthcheck task @@ -53,14 +51,18 @@ def create_embeddings(tenant_id, document_version_id): # Retrieve the Catalog ID doc = Document.query.get_or_404(document_version.doc_id) catalog_id = doc.catalog_id + catalog = Catalog.query.get_or_404(catalog_id) # Select variables to work with depending on tenant and model - model_variables = select_model_variables(tenant, catalog_id=catalog_id) - current_app.logger.debug(f'Model variables: {model_variables}') + model_variables = get_model_variables(tenant_id) + + # Define processor related information + processor_type, processor_class = ProcessorRegistry.get_processor_for_file_type(document_version.file_type) + processor = get_processor_for_document(catalog_id, document_version.file_type, document_version.sub_file_type) except Exception as e: current_app.logger.error(f'Create Embeddings request received ' - f'for non existing document version {document_version_id} ' + f'for badly configured document version {document_version_id} ' f'for tenant {tenant_id}, ' f'error: {e}') raise @@ -90,19 +92,19 @@ def create_embeddings(tenant_id, document_version_id): delete_embeddings_for_document_version(document_version) try: - match document_version.file_type: - case 'pdf': - process_pdf(tenant, model_variables, document_version) - case 'html': - process_html(tenant, model_variables, document_version) - case 'srt': - process_srt(tenant, model_variables, document_version) - case 'mp4' | 'mp3' | 'ogg': - process_audio(tenant, model_variables, document_version) - case _: - raise Exception(f'No functionality defined for file type {document_version.file_type} ' - f'for tenant {tenant_id} ' - f'while creating embeddings for document version {document_version_id}') + with current_event.create_span(f"{processor_type} Processing"): + document_processor = processor_class( + tenant=tenant, + model_variables=model_variables, + document_version=document_version, + catalog=catalog, + processor=processor + ) + markdown, title = document_processor.process() + + with current_event.create_span("Embedding"): + embed_markdown(tenant, model_variables, document_version, catalog, markdown, title) + current_event.log("Finished Embedding Creation Task") except Exception as e: @@ -129,53 +131,12 @@ def delete_embeddings_for_document_version(document_version): raise -def process_pdf(tenant, model_variables, document_version): - with current_event.create_span("PDF Processing"): - processor = PDFProcessor(tenant, model_variables, document_version) - markdown, title = processor.process() - - # Process markdown and embed - with current_event.create_span("Embedding"): - embed_markdown(tenant, model_variables, document_version, markdown, title) - - -def process_html(tenant, model_variables, document_version): - with current_event.create_span("HTML Processing"): - processor = HTMLProcessor(tenant, model_variables, document_version) - markdown, title = processor.process() - - # Process markdown and embed - with current_event.create_span("Embedding"): - embed_markdown(tenant, model_variables, document_version, markdown, title) - - -def process_audio(tenant, model_variables, document_version): - with current_event.create_span("Audio Processing"): - processor = AudioProcessor(tenant, model_variables, document_version) - markdown, title = processor.process() - - # Process markdown and embed - with current_event.create_span("Embedding"): - embed_markdown(tenant, model_variables, document_version, markdown, title) - - -def process_srt(tenant, model_variables, document_version): - with current_event.create_span("SRT Processing"): - processor = SRTProcessor(tenant, model_variables, document_version) - markdown, title = processor.process() - - # Process markdown and embed - with current_event.create_span("Embedding"): - embed_markdown(tenant, model_variables, document_version, markdown, title) - - -def embed_markdown(tenant, model_variables, document_version, markdown, title): +def embed_markdown(tenant, model_variables, document_version, catalog, markdown, title): # Create potential chunks potential_chunks = create_potential_chunks_for_markdown(tenant.id, document_version, f"{document_version.id}.md") # Combine chunks for embedding - chunks = combine_chunks_for_markdown(potential_chunks, model_variables['min_chunk_size'], - model_variables['max_chunk_size']) + chunks = combine_chunks_for_markdown(potential_chunks, catalog.min_chunk_size, catalog.max_chunk_size) # Enrich chunks with current_event.create_span("Enrich Chunks"): @@ -203,9 +164,6 @@ def embed_markdown(tenant, model_variables, document_version, markdown, title): def enrich_chunks(tenant, model_variables, document_version, title, chunks): - current_app.logger.debug(f'Enriching chunks for tenant {tenant.id} ' - f'on document version {document_version.id}') - summary = '' if len(chunks) > 1: summary = summarize_chunk(tenant, model_variables, document_version, chunks[0]) @@ -233,18 +191,13 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks): enriched_chunk = f'{chunk_total_context}\n{chunk}' enriched_chunks.append(enriched_chunk) - current_app.logger.debug(f'Finished enriching chunks for tenant {tenant.id} ' - f'on document version {document_version.id}') - return enriched_chunks def summarize_chunk(tenant, model_variables, document_version, chunk): current_event.log("Starting Summarizing Chunk") - current_app.logger.debug(f'Summarizing chunk for tenant {tenant.id} ' - f'on document version {document_version.id}') - llm = model_variables['llm'] - template = model_variables['summary_template'] + llm = model_variables.get_llm() + template = model_variables.get_template("summary") language_template = create_language_template(template, document_version.language) summary_prompt = ChatPromptTemplate.from_template(language_template) setup = RunnablePassthrough() @@ -253,11 +206,7 @@ def summarize_chunk(tenant, model_variables, document_version, chunk): chain = setup | summary_prompt | llm | output_parser try: - current_app.logger.debug(f'Starting summarizing chunk for tenant {tenant.id} ' - f'on document version {document_version.id}') summary = chain.invoke({"text": chunk}) - current_app.logger.debug(f'Finished summarizing chunk for tenant {tenant.id} ' - f'on document version {document_version.id}.') current_event.log("Finished Summarizing Chunk") return summary except LangChainException as e: @@ -268,14 +217,10 @@ def summarize_chunk(tenant, model_variables, document_version, chunk): def embed_chunks(tenant, model_variables, document_version, chunks): - current_app.logger.debug(f'Embedding chunks for tenant {tenant.id} ' - f'on document version {document_version.id}') - embedding_model = model_variables['embedding_model'] + embedding_model = model_variables.embedding_model try: embeddings = embedding_model.embed_documents(chunks) - current_app.logger.debug(f'Finished embedding chunks for tenant {tenant.id} ' - f'on document version {document_version.id}') except LangChainException as e: current_app.logger.error(f'Error creating embeddings for tenant {tenant.id} ' f'on document version {document_version.id} while calling OpenAI API' @@ -285,28 +230,16 @@ def embed_chunks(tenant, model_variables, document_version, chunks): # Add embeddings to the database new_embeddings = [] for chunk, embedding in zip(chunks, embeddings): - new_embedding = model_variables['embedding_db_model']() + new_embedding = model_variables.embedding_model_class() new_embedding.document_version = document_version new_embedding.active = True new_embedding.chunk = chunk new_embedding.embedding = embedding new_embeddings.append(new_embedding) - current_app.logger.debug(f'Finished embedding chunks for tenant {tenant.id} ') - return new_embeddings -def log_parsing_info(tenant, tags, included_elements, excluded_elements, excluded_classes, elements_to_parse): - if tenant.embed_tuning: - current_app.embed_tuning_logger.debug(f'Tags to parse: {tags}') - current_app.embed_tuning_logger.debug(f'Included Elements: {included_elements}') - current_app.embed_tuning_logger.debug(f'Excluded Elements: {excluded_elements}') - current_app.embed_tuning_logger.debug(f'Excluded Classes: {excluded_classes}') - current_app.embed_tuning_logger.debug(f'Found {len(elements_to_parse)} elements to parse') - current_app.embed_tuning_logger.debug(f'First element to parse: {elements_to_parse[0]}') - - def create_potential_chunks_for_markdown(tenant_id, document_version, input_file): try: current_app.logger.info(f'Creating potential chunks for tenant {tenant_id}') @@ -328,7 +261,6 @@ def create_potential_chunks_for_markdown(tenant_id, document_version, input_file md_header_splits = markdown_splitter.split_text(markdown) potential_chunks = [doc.page_content for doc in md_header_splits] - current_app.logger.debug(f'Created {len(potential_chunks)} potential chunks for tenant {tenant_id}') return potential_chunks except Exception as e: current_app.logger.error(f'Error creating potential chunks for tenant {tenant_id}, with error: {e}') @@ -361,3 +293,69 @@ def combine_chunks_for_markdown(potential_chunks, min_chars, max_chars): actual_chunks.append(current_chunk) return actual_chunks + + +def get_processor_for_document(catalog_id: int, file_type: str, sub_file_type: str = None) -> Processor: + """ + Get the appropriate processor for a document based on catalog_id, file_type and optional sub_file_type. + + Args: + catalog_id: ID of the catalog + file_type: Type of file (e.g., 'pdf', 'html') + sub_file_type: Optional sub-type for specialized processing + + Returns: + Processor instance + + Raises: + ValueError: If no matching processor is found + """ + try: + # Start with base query for catalog + query = Processor.query.filter_by(catalog_id=catalog_id) + + # Find processor type that handles this file type + matching_processor_type = None + for proc_type, config in PROCESSOR_TYPES.items(): + supported_types = config['file_types'] + if isinstance(supported_types, str): + supported_types = [t.strip() for t in supported_types.split(',')] + + if file_type in supported_types: + matching_processor_type = proc_type + break + + if not matching_processor_type: + raise ValueError(f"No processor type found for file type: {file_type}") + + # Add processor type condition + query = query.filter_by(type=matching_processor_type) + + # If sub_file_type is provided, add that condition + if sub_file_type: + query = query.filter_by(sub_file_type=sub_file_type) + else: + # If no sub_file_type, prefer processors without sub_file_type specification + query = query.filter(or_(Processor.sub_file_type.is_(None), + Processor.sub_file_type == '')) + + # Get the first matching processor + processor = query.first() + + if not processor: + if sub_file_type: + raise ValueError( + f"No processor found for catalog {catalog_id} of type {matching_processor_type}, " + f"file type {file_type}, sub-type {sub_file_type}" + ) + else: + raise ValueError( + f"No processor found for catalog {catalog_id}, " + f"file type {file_type}" + ) + + return processor + + except Exception as e: + current_app.logger.error(f"Error finding processor: {str(e)}") + raise diff --git a/integrations/Wordpress/eveai-chat-widget/eveai-chat_plugin.php b/integrations/Wordpress/eveai-chat-widget/eveai-chat_plugin.php index 7b918aa..a1e5e11 100644 --- a/integrations/Wordpress/eveai-chat-widget/eveai-chat_plugin.php +++ b/integrations/Wordpress/eveai-chat-widget/eveai-chat_plugin.php @@ -29,7 +29,8 @@ function eveai_chat_shortcode($atts) { 'domain' => '', 'language' => 'en', 'supported_languages' => 'en,fr,de,es', - 'server_url' => 'https://evie.askeveai.com' + 'server_url' => 'https://evie.askeveai.com', + 'specialist_id' => '1' // Added specialist_id parameter ); // Merge provided attributes with defaults @@ -42,6 +43,7 @@ function eveai_chat_shortcode($atts) { $language = sanitize_text_field($atts['language']); $supported_languages = sanitize_text_field($atts['supported_languages']); $server_url = esc_url_raw($atts['server_url']); + $specialist_id = sanitize_text_field($atts['specialist_id']); // Sanitize specialist_id // Generate a unique ID for this instance of the chat widget $chat_id = 'chat-container-' . uniqid(); @@ -55,7 +57,8 @@ function eveai_chat_shortcode($atts) { '$domain', '$language', '$supported_languages', - '$server_url' + '$server_url', + '$specialist_id' ); eveAI.initializeChat('$chat_id'); }); diff --git a/integrations/Wordpress/eveai-chat-widget/js/eveai-chat-widget.js b/integrations/Wordpress/eveai-chat-widget/js/eveai-chat-widget.js index 3dc82e8..a93f852 100644 --- a/integrations/Wordpress/eveai-chat-widget/js/eveai-chat-widget.js +++ b/integrations/Wordpress/eveai-chat-widget/js/eveai-chat-widget.js @@ -1,6 +1,6 @@ class EveAIChatWidget extends HTMLElement { static get observedAttributes() { - return ['tenant-id', 'api-key', 'domain', 'language', 'languages', 'server-url']; + return ['tenant-id', 'api-key', 'domain', 'language', 'languages', 'server-url', 'specialist-id']; } constructor() { @@ -14,6 +14,7 @@ class EveAIChatWidget extends HTMLElement { this.maxConnectionIdleTime = 1 * 60 * 60 * 1000; // 1 hours in milliseconds this.languages = [] this.room = null; + this.specialistId = null; console.log('EveAIChatWidget constructor called'); } @@ -89,6 +90,7 @@ class EveAIChatWidget extends HTMLElement { this.languages = languageAttr ? languageAttr.split(',') : []; this.serverUrl = this.getAttribute('server-url'); this.currentLanguage = this.language; + this.specialistId = this.getAttribute('specialist-id'); console.log('Updated attributes:', { tenantId: this.tenantId, apiKey: this.apiKey, @@ -96,7 +98,8 @@ class EveAIChatWidget extends HTMLElement { language: this.language, currentLanguage: this.currentLanguage, languages: this.languages, - serverUrl: this.serverUrl + serverUrl: this.serverUrl, + specialistId: this.specialistId }); } @@ -107,15 +110,17 @@ class EveAIChatWidget extends HTMLElement { const language = this.getAttribute('language'); const languages = this.getAttribute('languages'); const serverUrl = this.getAttribute('server-url'); + const specialistId = this.getAttribute('specialist-id') console.log('Checking if all attributes are set:', { tenantId, apiKey, domain, language, languages, - serverUrl + serverUrl, + specialistId }); - return tenantId && apiKey && domain && language && languages && serverUrl; + return tenantId && apiKey && domain && language && languages && serverUrl && specialistId; } createLanguageDropdown() { @@ -241,13 +246,13 @@ class EveAIChatWidget extends HTMLElement { this.socket.on('task_status', (data) => { console.log('Task status received:', data.status); console.log('Task ID received:', data.taskId); - console.log('Citations type:', typeof data.citations, 'Citations:', data.citations); + console.log('Citations type:', typeof data.results.citations, 'Citations:', data.results.citations); if (data.status === 'pending') { this.updateProgress(); setTimeout(() => this.checkTaskStatus(data.taskId), 1000); // Poll every second } else if (data.status === 'success') { - this.addBotMessage(data.answer, data.interaction_id, data.algorithm, data.citations); + this.addBotMessage(data.results.answer, data.interaction_id, data.algorithm, data.results.citations || []); this.clearProgress(); // Clear progress indicator when done } else { this.setStatusMessage('Failed to process message.'); @@ -450,15 +455,21 @@ toggleFeedback(thumbsUp, thumbsDown, feedback, interactionId) { const selectedLanguage = this.languageSelect.value; - console.log('Sending message to backend'); - this.socket.emit('user_message', { - tenantId: this.tenantId, + // Updated message structure to match specialist execution format + const messageData = { + tenantId: parseInt(this.tenantId), token: this.jwtToken, - message, - language: selectedLanguage, + specialistId: parseInt(this.specialistId), + arguments: { + language: selectedLanguage, + query: message + }, timezone: this.userTimezone - }); - this.setStatusMessage('Processing started ...') + }; + + console.log('Sending message to backend:', messageData); + this.socket.emit('user_message', messageData); + this.setStatusMessage('Processing started ...'); } toggleSendButton(isProcessing) { diff --git a/integrations/Wordpress/eveai-chat-widget/js/eveai-sdk.js b/integrations/Wordpress/eveai-chat-widget/js/eveai-sdk.js index c43707b..ddbb6d6 100644 --- a/integrations/Wordpress/eveai-chat-widget/js/eveai-sdk.js +++ b/integrations/Wordpress/eveai-chat-widget/js/eveai-sdk.js @@ -1,14 +1,15 @@ // static/js/eveai-sdk.js class EveAI { - constructor(tenantId, apiKey, domain, language, languages, serverUrl) { + constructor(tenantId, apiKey, domain, language, languages, serverUrl, specialistId) { this.tenantId = tenantId; this.apiKey = apiKey; this.domain = domain; this.language = language; this.languages = languages; this.serverUrl = serverUrl; + this.specialistId = specialistId; - console.log('EveAI constructor:', { tenantId, apiKey, domain, language, languages, serverUrl }); + console.log('EveAI constructor:', { tenantId, apiKey, domain, language, languages, serverUrl, specialistId }); } initializeChat(containerId) { @@ -23,6 +24,7 @@ class EveAI { chatWidget.setAttribute('language', this.language); chatWidget.setAttribute('languages', this.languages); chatWidget.setAttribute('server-url', this.serverUrl); + chatWidget.setAttribute('specialist-id', this.specialistId); }); } else { console.error('Container not found'); diff --git a/migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py b/migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py new file mode 100644 index 0000000..805153b --- /dev/null +++ b/migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py @@ -0,0 +1,31 @@ +"""Allow JSON specialist input and results for interactions + +Revision ID: 2370a17da7cb +Revises: 0c347651837c +Create Date: 2024-11-07 14:33:57.678861 + +""" +from alembic import op +import sqlalchemy as sa +import pgvector +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '2370a17da7cb' +down_revision = '0c347651837c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('interaction', sa.Column('specialist_arguments', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('interaction', sa.Column('specialist_results', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('interaction', 'specialist_results') + op.drop_column('interaction', 'specialist_arguments') + # ### end Alembic commands ### diff --git a/migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py b/migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py new file mode 100644 index 0000000..53af46c --- /dev/null +++ b/migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py @@ -0,0 +1,31 @@ +"""Add sub_file_type to processor and document_version + +Revision ID: 4bf121f340e5 +Revises: e54c830a88c9 +Create Date: 2024-11-11 13:03:23.265230 + +""" +from alembic import op +import sqlalchemy as sa +import pgvector + + +# revision identifiers, used by Alembic. +revision = '4bf121f340e5' +down_revision = 'e54c830a88c9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('document_version', sa.Column('sub_file_type', sa.String(length=50), nullable=True)) + op.add_column('processor', sa.Column('sub_file_type', sa.String(length=50), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('processor', 'sub_file_type') + op.drop_column('document_version', 'sub_file_type') + # ### end Alembic commands ### diff --git a/migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py b/migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py new file mode 100644 index 0000000..ca0c67b --- /dev/null +++ b/migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py @@ -0,0 +1,41 @@ +"""Removed obsolete interaction fields, added specialist relation to interaction + +Revision ID: 7872ffdbac6c +Revises: 2370a17da7cb +Create Date: 2024-11-07 15:14:12.263011 + +""" +from alembic import op +import sqlalchemy as sa +import pgvector + + +# revision identifiers, used by Alembic. +revision = '7872ffdbac6c' +down_revision = '2370a17da7cb' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('interaction', sa.Column('specialist_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'interaction', 'specialist', ['specialist_id'], ['id']) + op.drop_column('interaction', 'question') + op.drop_column('interaction', 'answer') + op.drop_column('interaction', 'algorithm_used') + op.drop_column('interaction', 'language') + op.drop_column('interaction', 'detailed_question') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('interaction', sa.Column('detailed_question', sa.TEXT(), autoincrement=False, nullable=True)) + op.add_column('interaction', sa.Column('language', sa.VARCHAR(length=2), autoincrement=False, nullable=False)) + op.add_column('interaction', sa.Column('algorithm_used', sa.VARCHAR(length=20), autoincrement=False, nullable=True)) + op.add_column('interaction', sa.Column('answer', sa.TEXT(), autoincrement=False, nullable=True)) + op.add_column('interaction', sa.Column('question', sa.TEXT(), autoincrement=False, nullable=False)) + op.drop_constraint(None, 'interaction', type_='foreignkey') + op.drop_column('interaction', 'specialist_id') + # ### end Alembic commands ### diff --git a/migrations/tenant/versions/95f82a301dff_adding_processor_model.py b/migrations/tenant/versions/95f82a301dff_adding_processor_model.py new file mode 100644 index 0000000..88df44d --- /dev/null +++ b/migrations/tenant/versions/95f82a301dff_adding_processor_model.py @@ -0,0 +1,49 @@ +"""Adding Processor Model + +Revision ID: 95f82a301dff +Revises: 7872ffdbac6c +Create Date: 2024-11-11 11:26:23.103689 + +""" +from alembic import op +import sqlalchemy as sa +import pgvector +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '95f82a301dff' +down_revision = '7872ffdbac6c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('processor', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=50), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('catalog_id', sa.Integer(), nullable=True), + sa.Column('type', sa.String(length=50), nullable=False), + sa.Column('min_chunk_size', sa.Integer(), nullable=True), + sa.Column('max_chunk_size', sa.Integer(), nullable=True), + sa.Column('tuning', sa.Boolean(), nullable=True), + sa.Column('user_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('system_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('configuration', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('created_by', sa.Integer(), nullable=True), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_by', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['catalog_id'], ['catalog.id'], ), + sa.ForeignKeyConstraint(['created_by'], ['public.user.id'], ), + sa.ForeignKeyConstraint(['updated_by'], ['public.user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('processor') + # ### end Alembic commands ### diff --git a/migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py b/migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py new file mode 100644 index 0000000..c3ee513 --- /dev/null +++ b/migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py @@ -0,0 +1,47 @@ +"""Remove obsolete columns due to Processor model introduction + +Revision ID: e54c830a88c9 +Revises: 95f82a301dff +Create Date: 2024-11-11 11:31:33.478714 + +""" +from alembic import op +import sqlalchemy as sa +import pgvector +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'e54c830a88c9' +down_revision = '95f82a301dff' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('catalog', 'chat_RAG_temperature') + op.drop_column('catalog', 'html_included_elements') + op.drop_column('catalog', 'html_tags') + op.drop_column('catalog', 'embed_tuning') + op.drop_column('catalog', 'html_end_tags') + op.drop_column('catalog', 'html_excluded_classes') + op.drop_column('catalog', 'chat_no_RAG_temperature') + op.drop_column('catalog', 'html_excluded_elements') + op.drop_column('processor', 'min_chunk_size') + op.drop_column('processor', 'max_chunk_size') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('processor', sa.Column('max_chunk_size', sa.INTEGER(), autoincrement=False, nullable=True)) + op.add_column('processor', sa.Column('min_chunk_size', sa.INTEGER(), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('html_excluded_elements', postgresql.ARRAY(sa.VARCHAR(length=50)), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('chat_no_RAG_temperature', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('html_excluded_classes', postgresql.ARRAY(sa.VARCHAR(length=200)), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('html_end_tags', postgresql.ARRAY(sa.VARCHAR(length=10)), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('embed_tuning', sa.BOOLEAN(), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('html_tags', postgresql.ARRAY(sa.VARCHAR(length=10)), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('html_included_elements', postgresql.ARRAY(sa.VARCHAR(length=50)), autoincrement=False, nullable=True)) + op.add_column('catalog', sa.Column('chat_RAG_temperature', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True)) + # ### end Alembic commands ### diff --git a/nginx/public/chat_evie.html b/nginx/public/chat_evie.html index d41af6f..9121a39 100644 --- a/nginx/public/chat_evie.html +++ b/nginx/public/chat_evie.html @@ -21,7 +21,8 @@ 'http://macstudio.ask-eve-ai-local.com', 'en', 'en,fr,nl', - 'http://macstudio.ask-eve-ai-local.com:8080/' + 'http://macstudio.ask-eve-ai-local.com:8080/', + '7' ); eveAI.initializeChat('chat-container'); }); diff --git a/requirements.txt b/requirements.txt index be8e3db..da7800c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,7 +63,6 @@ zxcvbn~=4.4.28 groq~=0.9.0 pydub~=0.25.1 argparse~=1.4.0 -portkey_ai~=1.8.7 minio~=7.2.7 Werkzeug~=3.0.3 itsdangerous~=2.2.0 @@ -83,3 +82,10 @@ flower~=2.0.1 psutil~=6.0.0 celery-redbeat~=2.2.0 WTForms-SQLAlchemy~=0.4.1 +packaging~=24.1 + +typing_extensions~=4.12.2 +prometheus_flask_exporter~=0.23.1 +prometheus_client~=0.20.0 +babel~=2.16.0 +dogpile.cache~=1.3.3 \ No newline at end of file