pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Introduction of dynamic Retrievers & Specialists

Browse Source 
- Introduction of dynamic Processors
- Introduction of caching system
- Introduction of a better template manager
- Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists
- Start adaptation of chat client
This commit is contained in:
Josako
2024-11-15 10:00:53 +01:00
parent 55a8a95f79
commit 1807435339
101 changed files with 4181 additions and 1764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -43,3 +43,5 @@ scripts/.DS_Store
scripts/__pycache__/run_eveai_app.cpython-312.pyc scripts/__pycache__/run_eveai_app.cpython-312.pyc
/eveai_repo.txt /eveai_repo.txt
*repo.txt *repo.txt
/docker/eveai_logs/
/common/utils/model_utils_orig.py

4
common/extensions.py
View File

@@ -12,6 +12,8 @@ from flask_wtf import CSRFProtect
from flask_restx import Api from flask_restx import Api
from prometheus_flask_exporter import PrometheusMetrics from prometheus_flask_exporter import PrometheusMetrics
from .langchain.templates.template_manager import TemplateManager
from .utils.cache.eveai_cache_manager import EveAICacheManager
from .utils.simple_encryption import SimpleEncryption from .utils.simple_encryption import SimpleEncryption
from .utils.minio_utils import MinioClient from .utils.minio_utils import MinioClient
@@ -32,3 +34,5 @@ api_rest = Api()
simple_encryption = SimpleEncryption() simple_encryption = SimpleEncryption()
minio_client = MinioClient() minio_client = MinioClient()
metrics = PrometheusMetrics.for_app_factory() metrics = PrometheusMetrics.for_app_factory()
template_manager = TemplateManager()
cache_manager = EveAICacheManager()

23
common/langchain/outputs/base.py Normal file
View File

@@ -0,0 +1,23 @@
# Output Schema Management - common/langchain/outputs/base.py
from typing import Dict, Type, Any
from pydantic import BaseModel
class BaseSpecialistOutput(BaseModel):
"""Base class for all specialist outputs"""
pass
class OutputRegistry:
"""Registry for specialist output schemas"""
_schemas: Dict[str, Type[BaseSpecialistOutput]] = {}
@classmethod
def register(cls, specialist_type: str, schema_class: Type[BaseSpecialistOutput]):
cls._schemas[specialist_type] = schema_class
@classmethod
def get_schema(cls, specialist_type: str) -> Type[BaseSpecialistOutput]:
if specialist_type not in cls._schemas:
raise ValueError(f"No output schema registered for {specialist_type}")
return cls._schemas[specialist_type]

22
common/langchain/outputs/rag.py Normal file
View File

@@ -0,0 +1,22 @@
# RAG Specialist Output - common/langchain/outputs/rag.py
from typing import List
from pydantic import Field
from .base import BaseSpecialistOutput
class RAGOutput(BaseSpecialistOutput):
"""Output schema for RAG specialist"""
"""Default docstring - to be replaced with actual prompt"""
answer: str = Field(
...,
description="The answer to the user question, based on the given sources",
)
citations: List[int] = Field(
...,
description="The integer IDs of the SPECIFIC sources that were used to generate the answer"
)
insufficient_info: bool = Field(
False, # Default value is set to False
description="A boolean indicating whether given sources were sufficient or not to generate the answer"
)

145
common/langchain/retrievers/eveai_default_rag_retriever.py
View File

@@ -1,145 +0,0 @@
from langchain_core.retrievers import BaseRetriever
from sqlalchemy import func, and_, or_, desc
from sqlalchemy.exc import SQLAlchemyError
from pydantic import BaseModel, Field, PrivateAttr
from typing import Any, Dict
from flask import current_app
from common.extensions import db
from common.models.document import Document, DocumentVersion
from common.utils.datetime_utils import get_date_in_timezone
from common.utils.model_utils import ModelVariables
class EveAIDefaultRagRetriever(BaseRetriever, BaseModel):
_catalog_id: int = PrivateAttr()
_model_variables: ModelVariables = PrivateAttr()
_tenant_info: Dict[str, Any] = PrivateAttr()
def __init__(self, catalog_id: int, model_variables: ModelVariables, tenant_info: Dict[str, Any]):
super().__init__()
current_app.logger.debug(f'Model variables type: {type(model_variables)}')
self._catalog_id = catalog_id
self._model_variables = model_variables
self._tenant_info = tenant_info
@property
def catalog_id(self) -> int:
return self._catalog_id
@property
def model_variables(self) -> ModelVariables:
return self._model_variables
@property
def tenant_info(self) -> Dict[str, Any]:
return self._tenant_info
def _get_relevant_documents(self, query: str):
current_app.logger.debug(f'Retrieving relevant documents for query: {query}')
query_embedding = self._get_query_embedding(query)
current_app.logger.debug(f'Model Variables Private: {type(self._model_variables)}')
current_app.logger.debug(f'Model Variables Property: {type(self.model_variables)}')
db_class = self.model_variables['embedding_db_model']
similarity_threshold = self.model_variables['similarity_threshold']
k = self.model_variables['k']
if self.model_variables['rag_tuning']:
try:
current_date = get_date_in_timezone(self.tenant_info['timezone'])
current_app.rag_tuning_logger.debug(f'Current date: {current_date}\n')
# Debug query to show similarity for all valid documents (without chunk text)
debug_query = (
db.session.query(
Document.id.label('document_id'),
DocumentVersion.id.label('version_id'),
db_class.id.label('embedding_id'),
(1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity')
)
.join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
.join(Document, DocumentVersion.doc_id == Document.id)
.filter(
or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date)
)
.order_by(desc('similarity'))
)
debug_results = debug_query.all()
current_app.logger.debug("Debug: Similarity for all valid documents:")
for row in debug_results:
current_app.rag_tuning_logger.debug(f"Doc ID: {row.document_id}, "
f"Version ID: {row.version_id}, "
f"Embedding ID: {row.embedding_id}, "
f"Similarity: {row.similarity}")
current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
except SQLAlchemyError as e:
current_app.logger.error(f'Error generating overview: {e}')
db.session.rollback()
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Parameters for Retrieval of documents: \n')
current_app.rag_tuning_logger.debug(f'Similarity Threshold: {similarity_threshold}\n')
current_app.rag_tuning_logger.debug(f'K: {k}\n')
current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
try:
current_date = get_date_in_timezone(self.tenant_info['timezone'])
# Subquery to find the latest version of each document
subquery = (
db.session.query(
DocumentVersion.doc_id,
func.max(DocumentVersion.id).label('latest_version_id')
)
.group_by(DocumentVersion.doc_id)
.subquery()
)
# Main query to filter embeddings
query_obj = (
db.session.query(db_class,
(1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity'))
.join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
.join(Document, DocumentVersion.doc_id == Document.id)
.join(subquery, DocumentVersion.id == subquery.c.latest_version_id)
.filter(
or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date),
(1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold,
Document.catalog_id == self._catalog_id
)
.order_by(desc('similarity'))
.limit(k)
)
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Query executed for Retrieval of documents: \n')
current_app.rag_tuning_logger.debug(f'{query_obj.statement}\n')
current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
res = query_obj.all()
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Retrieved {len(res)} relevant documents \n')
current_app.rag_tuning_logger.debug(f'Data retrieved: \n')
current_app.rag_tuning_logger.debug(f'{res}\n')
current_app.rag_tuning_logger.debug(f'---------------------------------------\n')
result = []
for doc in res:
if self.model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n')
current_app.rag_tuning_logger.debug(f'Chunk: \n {doc[0].chunk}\n\n')
result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n')
except SQLAlchemyError as e:
current_app.logger.error(f'Error retrieving relevant documents: {e}')
db.session.rollback()
return []
return result
def _get_query_embedding(self, query: str):
embedding_model = self.model_variables['embedding_model']
query_embedding = embedding_model.embed_query(query)
return query_embedding

154
common/langchain/retrievers/eveai_dossier_retriever.py
View File

@@ -1,154 +0,0 @@
from langchain_core.retrievers import BaseRetriever
from sqlalchemy import func, and_, or_, desc, cast, JSON
from sqlalchemy.exc import SQLAlchemyError
from pydantic import BaseModel, Field, PrivateAttr
from typing import Any, Dict, List, Optional
from flask import current_app
from contextlib import contextmanager
from common.extensions import db
from common.models.document import Document, DocumentVersion, Catalog
from common.utils.datetime_utils import get_date_in_timezone
from common.utils.model_utils import ModelVariables
class EveAIDossierRetriever(BaseRetriever, BaseModel):
_catalog_id: int = PrivateAttr()
_model_variables: ModelVariables = PrivateAttr()
_tenant_info: Dict[str, Any] = PrivateAttr()
_active_filters: Optional[Dict[str, Any]] = PrivateAttr()
def __init__(self, catalog_id: int, model_variables: ModelVariables, tenant_info: Dict[str, Any]):
super().__init__()
self._catalog_id = catalog_id
self._model_variables = model_variables
self._tenant_info = tenant_info
self._active_filters = None
@contextmanager
def filtering(self, metadata_filters: Dict[str, Any]):
"""Context manager for temporarily setting metadata filters"""
previous_filters = self._active_filters
self._active_filters = metadata_filters
try:
yield self
finally:
self._active_filters = previous_filters
def _build_metadata_filter_conditions(self, query):
"""Build SQL conditions for metadata filtering"""
if not self._active_filters:
return query
conditions = []
for field, value in self._active_filters.items():
if value is None:
continue
# Handle both single values and lists of values
if isinstance(value, (list, tuple)):
# Multiple values - create OR condition
or_conditions = []
for val in value:
or_conditions.append(
cast(DocumentVersion.user_metadata[field].astext, JSON) == str(val)
)
if or_conditions:
conditions.append(or_(*or_conditions))
else:
# Single value - direct comparison
conditions.append(
cast(DocumentVersion.user_metadata[field].astext, JSON) == str(value)
)
if conditions:
query = query.filter(and_(*conditions))
return query
def _get_relevant_documents(self, query: str):
current_app.logger.debug(f'Retrieving relevant documents for dossier query: {query}')
if self._active_filters:
current_app.logger.debug(f'Using metadata filters: {self._active_filters}')
query_embedding = self._get_query_embedding(query)
db_class = self.model_variables['embedding_db_model']
similarity_threshold = self.model_variables['similarity_threshold']
k = self.model_variables['k']
try:
current_date = get_date_in_timezone(self.tenant_info['timezone'])
# Subquery to find the latest version of each document
subquery = (
db.session.query(
DocumentVersion.doc_id,
func.max(DocumentVersion.id).label('latest_version_id')
)
.group_by(DocumentVersion.doc_id)
.subquery()
)
# Build base query
# Build base query
query_obj = (
db.session.query(db_class,
(1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity'))
.join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
.join(Document, DocumentVersion.doc_id == Document.id)
.join(subquery, DocumentVersion.id == subquery.c.latest_version_id)
.filter(
or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date),
(1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold,
Document.catalog_id == self._catalog_id
)
)
# Apply metadata filters
query_obj = self._build_metadata_filter_conditions(query_obj)
# Order and limit results
query_obj = query_obj.order_by(desc('similarity')).limit(k)
# Debug logging for RAG tuning if enabled
if self.model_variables['rag_tuning']:
self._log_rag_tuning(query_obj, query_embedding)
res = query_obj.all()
result = []
for doc in res:
if self.model_variables['rag_tuning']:
current_app.logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n')
current_app.logger.debug(f'Chunk: \n {doc[0].chunk}\n\n')
result.append(f'SOURCE: {doc[0].id}\n\n{doc[0].chunk}\n\n')
except SQLAlchemyError as e:
current_app.logger.error(f'Error retrieving relevant documents: {e}')
db.session.rollback()
return []
return result
def _log_rag_tuning(self, query_obj, query_embedding):
"""Log debug information for RAG tuning"""
current_app.rag_tuning_logger.debug("Debug: Query execution plan:")
current_app.rag_tuning_logger.debug(f"{query_obj.statement}")
if self._active_filters:
current_app.rag_tuning_logger.debug("Debug: Active metadata filters:")
current_app.rag_tuning_logger.debug(f"{self._active_filters}")
def _get_query_embedding(self, query: str):
"""Get embedding for the query text"""
embedding_model = self.model_variables['embedding_model']
query_embedding = embedding_model.embed_query(query)
return query_embedding
@property
def model_variables(self) -> ModelVariables:
return self._model_variables
@property
def tenant_info(self) -> Dict[str, Any]:
return self._tenant_info

52
common/langchain/retrievers/eveai_history_retriever.py
View File

@@ -1,52 +0,0 @@
from langchain_core.retrievers import BaseRetriever
from sqlalchemy import asc
from sqlalchemy.exc import SQLAlchemyError
from pydantic import Field, BaseModel, PrivateAttr
from typing import Any, Dict
from flask import current_app
from common.extensions import db
from common.models.interaction import ChatSession, Interaction
from common.utils.model_utils import ModelVariables
class EveAIHistoryRetriever(BaseRetriever, BaseModel):
_model_variables: ModelVariables = PrivateAttr()
_session_id: str = PrivateAttr()
def __init__(self, model_variables: ModelVariables, session_id: str):
super().__init__()
self._model_variables = model_variables
self._session_id = session_id
@property
def model_variables(self) -> ModelVariables:
return self._model_variables
@property
def session_id(self) -> str:
return self._session_id
def _get_relevant_documents(self, query: str):
current_app.logger.debug(f'Retrieving history of interactions for query: {query}')
try:
query_obj = (
db.session.query(Interaction)
.join(ChatSession, Interaction.chat_session_id == ChatSession.id)
.filter(ChatSession.session_id == self.session_id)
.order_by(asc(Interaction.id))
)
interactions = query_obj.all()
result = []
for interaction in interactions:
result.append(f'HUMAN:\n{interaction.detailed_question}\n\nAI: \n{interaction.answer}\n\n')
except SQLAlchemyError as e:
current_app.logger.error(f'Error retrieving history of interactions: {e}')
db.session.rollback()
return []
return result

40
common/langchain/retrievers/eveai_retriever.py
View File

@@ -1,40 +0,0 @@
from pydantic import BaseModel, PrivateAttr
from typing import Dict, Any
from common.utils.model_utils import ModelVariables
class EveAIRetriever(BaseModel):
_catalog_id: int = PrivateAttr()
_user_metadata: Dict[str, Any] = PrivateAttr()
_system_metadata: Dict[str, Any] = PrivateAttr()
_configuration: Dict[str, Any] = PrivateAttr()
_tenant_info: Dict[str, Any] = PrivateAttr()
_model_variables: ModelVariables = PrivateAttr()
_tuning: bool = PrivateAttr()
def __init__(self, catalog_id: int, user_metadata: Dict[str, Any], system_metadata: Dict[str, Any],
configuration: Dict[str, Any]):
super().__init__()
self._catalog_id = catalog_id
self._user_metadata = user_metadata
self._system_metadata = system_metadata
self._configuration = configuration
@property
def catalog_id(self):
return self._catalog_id
@property
def user_metadata(self):
return self._user_metadata
@property
def system_metadata(self):
return self._system_metadata
@property
def configuration(self):
return self._configuration
# Any common methods that should be shared among retrievers can go here.

154
common/langchain/templates/template_manager.py Normal file
View File

@@ -0,0 +1,154 @@
import os
import yaml
from typing import Dict, Optional, Any
from packaging import version
from dataclasses import dataclass
from flask import current_app, Flask
from common.utils.os_utils import get_project_root
@dataclass
class PromptTemplate:
"""Represents a versioned prompt template"""
content: str
version: str
metadata: Dict[str, Any]
class TemplateManager:
"""Manages versioned prompt templates"""
def __init__(self):
self.templates_dir = None
self._templates = None
self.app = None
def init_app(self, app: Flask) -> None:
# Initialize template manager
base_dir = "/app"
self.templates_dir = os.path.join(base_dir, 'config', 'prompts')
app.logger.debug(f'Loading templates from {self.templates_dir}')
self.app = app
self._templates = self._load_templates()
# Log available templates for each supported model
for llm in app.config['SUPPORTED_LLMS']:
try:
available_templates = self.list_templates(llm)
app.logger.info(f"Loaded templates for {llm}: {available_templates}")
except ValueError:
app.logger.warning(f"No templates found for {llm}")
def _load_templates(self) -> Dict[str, Dict[str, Dict[str, PromptTemplate]]]:
"""
Load all template versions from the templates directory.
Structure: {provider.model -> {template_name -> {version -> template}}}
Directory structure:
prompts/
├── provider/
│ └── model/
│ └── template_name/
│ └── version.yaml
"""
templates = {}
# Iterate through providers (anthropic, openai)
for provider in os.listdir(self.templates_dir):
provider_path = os.path.join(self.templates_dir, provider)
if not os.path.isdir(provider_path):
continue
# Iterate through models (claude-3, gpt-4o)
for model in os.listdir(provider_path):
model_path = os.path.join(provider_path, model)
if not os.path.isdir(model_path):
continue
provider_model = f"{provider}.{model}"
templates[provider_model] = {}
# Iterate through template types (rag, summary, etc.)
for template_name in os.listdir(model_path):
template_path = os.path.join(model_path, template_name)
if not os.path.isdir(template_path):
continue
template_versions = {}
# Load all version files for this template
for version_file in os.listdir(template_path):
if not version_file.endswith('.yaml'):
continue
version_str = version_file[:-5] # Remove .yaml
if not self._is_valid_version(version_str):
current_app.logger.warning(
f"Invalid version format for {template_name}: {version_str}")
continue
try:
with open(os.path.join(template_path, version_file)) as f:
template_data = yaml.safe_load(f)
# Verify required fields
if not template_data.get('content'):
raise ValueError("Template content is required")
template_versions[version_str] = PromptTemplate(
content=template_data['content'],
version=version_str,
metadata=template_data.get('metadata', {})
)
except Exception as e:
current_app.logger.error(
f"Error loading template {template_name} version {version_str}: {e}")
continue
if template_versions:
templates[provider_model][template_name] = template_versions
return templates
def _is_valid_version(self, version_str: str) -> bool:
"""Validate semantic versioning string"""
try:
version.parse(version_str)
return True
except version.InvalidVersion:
return False
def get_template(self,
provider_model: str,
template_name: str,
template_version: Optional[str] = None) -> PromptTemplate:
"""
Get a specific template version. If version not specified,
returns the latest version.
"""
if provider_model not in self._templates:
raise ValueError(f"Unknown provider.model: {provider_model}")
if template_name not in self._templates[provider_model]:
raise ValueError(f"Unknown template: {template_name}")
versions = self._templates[provider_model][template_name]
if template_version:
if template_version not in versions:
raise ValueError(f"Template version {template_version} not found")
return versions[template_version]
# Return latest version
latest = max(versions.keys(), key=version.parse)
return versions[latest]
def list_templates(self, provider_model: str) -> Dict[str, list]:
"""
List all available templates and their versions for a provider.model
Returns: {template_name: [version1, version2, ...]}
"""
if provider_model not in self._templates:
raise ValueError(f"Unknown provider.model: {provider_model}")
return {
template_name: sorted(versions.keys(), key=version.parse)
for template_name, versions in self._templates[provider_model].items()
}

27
common/langchain/tracked_transcribe.py
View File

@@ -1,27 +0,0 @@
import time
from common.utils.business_event_context import current_event
def tracked_transcribe(client, *args, **kwargs):
start_time = time.time()
# Extract the file and model from kwargs if present, otherwise use defaults
file = kwargs.get('file')
model = kwargs.get('model', 'whisper-1')
duration = kwargs.pop('duration', 600)
result = client.audio.transcriptions.create(*args, **kwargs)
end_time = time.time()
# Token usage for transcriptions is actually the duration in seconds we pass, as the whisper model is priced per second transcribed
metrics = {
'total_tokens': duration,
'prompt_tokens': 0, # For transcriptions, all tokens are considered "completion"
'completion_tokens': duration,
'time_elapsed': end_time - start_time,
'interaction_type': 'ASR',
}
current_event.log_llm_metrics(metrics)
return result

77
common/langchain/tracked_transcription.py Normal file
View File

@@ -0,0 +1,77 @@
# common/langchain/tracked_transcription.py
from typing import Any, Optional, Dict
import time
from openai import OpenAI
from common.utils.business_event_context import current_event
class TrackedOpenAITranscription:
"""Wrapper for OpenAI transcription with metric tracking"""
def __init__(self, api_key: str, **kwargs: Any):
"""Initialize with OpenAI client settings"""
self.client = OpenAI(api_key=api_key)
self.model = kwargs.get('model', 'whisper-1')
def transcribe(self,
file: Any,
model: Optional[str] = None,
language: Optional[str] = None,
prompt: Optional[str] = None,
response_format: Optional[str] = None,
temperature: Optional[float] = None,
duration: Optional[int] = None) -> str:
"""
Transcribe audio with metrics tracking
Args:
file: Audio file to transcribe
model: Model to use (defaults to whisper-1)
language: Optional language of the audio
prompt: Optional prompt to guide transcription
response_format: Response format (json, text, etc)
temperature: Sampling temperature
duration: Duration of audio in seconds for metrics
Returns:
Transcription text
"""
start_time = time.time()
try:
# Create transcription options
options = {
"file": file,
"model": model or self.model,
}
if language:
options["language"] = language
if prompt:
options["prompt"] = prompt
if response_format:
options["response_format"] = response_format
if temperature:
options["temperature"] = temperature
response = self.client.audio.transcriptions.create(**options)
# Calculate metrics
end_time = time.time()
# Token usage for transcriptions is based on audio duration
metrics = {
'total_tokens': duration or 600, # Default to 10 minutes if duration not provided
'prompt_tokens': 0, # For transcriptions, all tokens are completion
'completion_tokens': duration or 600,
'time_elapsed': end_time - start_time,
'interaction_type': 'ASR',
}
current_event.log_llm_metrics(metrics)
# Return text from response
if isinstance(response, str):
return response
return response.text
except Exception as e:
raise Exception(f"Transcription failed: {str(e)}")

32
common/models/document.py
View File

@@ -12,22 +12,31 @@ class Catalog(db.Model):
description = db.Column(db.Text, nullable=True) description = db.Column(db.Text, nullable=True)
type = db.Column(db.String(50), nullable=False, default="STANDARD_CATALOG") type = db.Column(db.String(50), nullable=False, default="STANDARD_CATALOG")
# Embedding variables
html_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
html_end_tags = db.Column(ARRAY(sa.String(10)), nullable=True, default=['p', 'li'])
html_included_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
html_excluded_elements = db.Column(ARRAY(sa.String(50)), nullable=True)
html_excluded_classes = db.Column(ARRAY(sa.String(200)), nullable=True)
min_chunk_size = db.Column(db.Integer, nullable=True, default=2000) min_chunk_size = db.Column(db.Integer, nullable=True, default=2000)
max_chunk_size = db.Column(db.Integer, nullable=True, default=3000) max_chunk_size = db.Column(db.Integer, nullable=True, default=3000)
# Chat variables ==> Move to Specialist? # Meta Data
chat_RAG_temperature = db.Column(db.Float, nullable=True, default=0.3) user_metadata = db.Column(JSONB, nullable=True)
chat_no_RAG_temperature = db.Column(db.Float, nullable=True, default=0.5) system_metadata = db.Column(JSONB, nullable=True)
configuration = db.Column(JSONB, nullable=True)
# Versioning Information
created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=True)
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
class Processor(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(50), nullable=False)
description = db.Column(db.Text, nullable=True)
catalog_id = db.Column(db.Integer, db.ForeignKey('catalog.id'), nullable=True)
type = db.Column(db.String(50), nullable=False)
sub_file_type = db.Column(db.String(50), nullable=True)
# Tuning enablers # Tuning enablers
embed_tuning = db.Column(db.Boolean, nullable=True, default=False) tuning = db.Column(db.Boolean, nullable=True, default=False)
# Meta Data # Meta Data
user_metadata = db.Column(JSONB, nullable=True) user_metadata = db.Column(JSONB, nullable=True)
@@ -90,6 +99,7 @@ class DocumentVersion(db.Model):
bucket_name = db.Column(db.String(255), nullable=True) bucket_name = db.Column(db.String(255), nullable=True)
object_name = db.Column(db.String(200), nullable=True) object_name = db.Column(db.String(200), nullable=True)
file_type = db.Column(db.String(20), nullable=True) file_type = db.Column(db.String(20), nullable=True)
sub_file_type = db.Column(db.String(50), nullable=True)
file_size = db.Column(db.Float, nullable=True) file_size = db.Column(db.Float, nullable=True)
language = db.Column(db.String(2), nullable=False) language = db.Column(db.String(2), nullable=False)
user_context = db.Column(db.Text, nullable=True) user_context = db.Column(db.Text, nullable=True)

55
common/models/interaction.py
View File

@@ -20,34 +20,6 @@ class ChatSession(db.Model):
return f"<ChatSession {self.id} by {self.user_id}>" return f"<ChatSession {self.id} by {self.user_id}>"
class Interaction(db.Model):
id = db.Column(db.Integer, primary_key=True)
chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=False)
question = db.Column(db.Text, nullable=False)
detailed_question = db.Column(db.Text, nullable=True)
answer = db.Column(db.Text, nullable=True)
algorithm_used = db.Column(db.String(20), nullable=True)
language = db.Column(db.String(2), nullable=False)
timezone = db.Column(db.String(30), nullable=True)
appreciation = db.Column(db.Integer, nullable=True)
# Timing information
question_at = db.Column(db.DateTime, nullable=False)
detailed_question_at = db.Column(db.DateTime, nullable=True)
answer_at = db.Column(db.DateTime, nullable=True)
# Relations
embeddings = db.relationship('InteractionEmbedding', backref='interaction', lazy=True)
def __repr__(self):
return f"<Interaction {self.id}>"
class InteractionEmbedding(db.Model):
interaction_id = db.Column(db.Integer, db.ForeignKey(Interaction.id, ondelete='CASCADE'), primary_key=True)
embedding_id = db.Column(db.Integer, db.ForeignKey(Embedding.id, ondelete='CASCADE'), primary_key=True)
class Specialist(db.Model): class Specialist(db.Model):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(50), nullable=False) name = db.Column(db.String(50), nullable=False)
@@ -68,7 +40,34 @@ class Specialist(db.Model):
updated_by = db.Column(db.Integer, db.ForeignKey(User.id)) updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
class Interaction(db.Model):
id = db.Column(db.Integer, primary_key=True)
chat_session_id = db.Column(db.Integer, db.ForeignKey(ChatSession.id), nullable=False)
specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id), nullable=True)
specialist_arguments = db.Column(JSONB, nullable=True)
specialist_results = db.Column(JSONB, nullable=True)
timezone = db.Column(db.String(30), nullable=True)
appreciation = db.Column(db.Integer, nullable=True)
# Timing information
question_at = db.Column(db.DateTime, nullable=False)
detailed_question_at = db.Column(db.DateTime, nullable=True)
answer_at = db.Column(db.DateTime, nullable=True)
# Relations
embeddings = db.relationship('InteractionEmbedding', backref='interaction', lazy=True)
def __repr__(self):
return f"<Interaction {self.id}>"
class InteractionEmbedding(db.Model):
interaction_id = db.Column(db.Integer, db.ForeignKey(Interaction.id, ondelete='CASCADE'), primary_key=True)
embedding_id = db.Column(db.Integer, db.ForeignKey(Embedding.id, ondelete='CASCADE'), primary_key=True)
class SpecialistRetriever(db.Model): class SpecialistRetriever(db.Model):
specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True) specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True)
retriever_id = db.Column(db.Integer, db.ForeignKey(Retriever.id, ondelete='CASCADE'), primary_key=True) retriever_id = db.Column(db.Integer, db.ForeignKey(Retriever.id, ondelete='CASCADE'), primary_key=True)
retriever = db.relationship("Retriever", backref="specialist_retrievers")

1
common/utils/business_event.py
View File

@@ -4,7 +4,6 @@ from contextlib import contextmanager
from datetime import datetime from datetime import datetime
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from datetime import datetime as dt, timezone as tz from datetime import datetime as dt, timezone as tz
from portkey_ai import Portkey, Config
import logging import logging
from .business_event_context import BusinessEventContext from .business_event_context import BusinessEventContext

0
common/langchain/retrievers/__init__.py → common/utils/cache/__init__old.py vendored
View File

89
common/utils/cache/base.py vendored Normal file
View File

@@ -0,0 +1,89 @@
# common/utils/cache/base.py
from typing import Any, Dict, List, Optional, TypeVar, Generic, Type
from dataclasses import dataclass
from flask import Flask
from dogpile.cache import CacheRegion
T = TypeVar('T')
@dataclass
class CacheKey:
"""Represents a cache key with multiple components"""
components: Dict[str, Any]
def __str__(self) -> str:
return ":".join(f"{k}={v}" for k, v in sorted(self.components.items()))
class CacheInvalidationManager:
"""Manages cache invalidation subscriptions"""
def __init__(self):
self._subscribers = {}
def subscribe(self, model: str, handler: 'CacheHandler', key_fields: List[str]):
if model not in self._subscribers:
self._subscribers[model] = []
self._subscribers[model].append((handler, key_fields))
def notify_change(self, model: str, **identifiers):
if model in self._subscribers:
for handler, key_fields in self._subscribers[model]:
if all(field in identifiers for field in key_fields):
handler.invalidate_by_model(model, **identifiers)
class CacheHandler(Generic[T]):
"""Base cache handler implementation"""
def __init__(self, region: CacheRegion, prefix: str):
self.region = region
self.prefix = prefix
self._key_components = []
def configure_keys(self, *components: str):
self._key_components = components
return self
def subscribe_to_model(self, model: str, key_fields: List[str]):
invalidation_manager.subscribe(model, self, key_fields)
return self
def generate_key(self, **identifiers) -> str:
missing = set(self._key_components) - set(identifiers.keys())
if missing:
raise ValueError(f"Missing key components: {missing}")
key = CacheKey({k: identifiers[k] for k in self._key_components})
return f"{self.prefix}:{str(key)}"
def get(self, creator_func, **identifiers) -> T:
cache_key = self.generate_key(**identifiers)
def creator():
instance = creator_func(**identifiers)
return self.to_cache_data(instance)
cached_data = self.region.get_or_create(
cache_key,
creator,
should_cache_fn=self.should_cache
)
return self.from_cache_data(cached_data, **identifiers)
def invalidate(self, **identifiers):
cache_key = self.generate_key(**identifiers)
self.region.delete(cache_key)
def invalidate_by_model(self, model: str, **identifiers):
try:
self.invalidate(**identifiers)
except ValueError:
pass
# Create global invalidation manager
invalidation_manager = CacheInvalidationManager()

32
common/utils/cache/eveai_cache_manager.py vendored Normal file
View File

@@ -0,0 +1,32 @@
from typing import Type
from flask import Flask
from common.utils.cache.base import CacheHandler
class EveAICacheManager:
"""Cache manager with registration capabilities"""
def __init__(self):
self.model_region = None
self.eveai_chat_workers_region = None
self.eveai_workers_region = None
self._handlers = {}
def init_app(self, app: Flask):
"""Initialize cache regions"""
from common.utils.cache.regions import create_cache_regions
self.model_region, self.eveai_chat_workers_region, self.eveai_workers_region = create_cache_regions(app)
# Initialize all registered handlers with their regions
for handler_class, region_name in self._handlers.items():
region = getattr(self, f"{region_name}_region")
handler_instance = handler_class(region)
setattr(self, handler_class.handler_name, handler_instance)
def register_handler(self, handler_class: Type[CacheHandler], region: str):
"""Register a cache handler class with its region"""
if not hasattr(handler_class, 'handler_name'):
raise ValueError("Cache handler must define handler_name class attribute")
self._handlers[handler_class] = region

61
common/utils/cache/regions.py vendored Normal file
View File

@@ -0,0 +1,61 @@
# common/utils/cache/regions.py
from dogpile.cache import make_region
from flask import current_app
from urllib.parse import urlparse
import os
def get_redis_config(app):
"""
Create Redis configuration dict based on app config
Handles both authenticated and non-authenticated setups
"""
# Parse the REDIS_BASE_URI to get all components
redis_uri = urlparse(app.config['REDIS_BASE_URI'])
config = {
'host': redis_uri.hostname,
'port': int(redis_uri.port or 6379),
'db': 4, # Keep this for later use
'redis_expiration_time': 3600,
'distributed_lock': True
}
# Add authentication if provided
if redis_uri.username and redis_uri.password:
config.update({
'username': redis_uri.username,
'password': redis_uri.password
})
return config
def create_cache_regions(app):
"""Initialize all cache regions with app config"""
redis_config = get_redis_config(app)
# Region for model-related caching (ModelVariables etc)
model_region = make_region(name='model').configure(
'dogpile.cache.redis',
arguments=redis_config,
replace_existing_backend=True
)
# Region for eveai_chat_workers components (Specialists, Retrievers, ...)
eveai_chat_workers_region = make_region(name='chat_workers').configure(
'dogpile.cache.redis',
arguments=redis_config, # arguments={**redis_config, 'db': 4}, # Different DB
replace_existing_backend=True
)
# Region for eveai_workers components (Processors, ...)
eveai_workers_region = make_region(name='workers').configure(
'dogpile.cache.redis',
arguments=redis_config, # Same config for now
replace_existing_backend=True
)
return model_region, eveai_chat_workers_region, eveai_workers_region

2
common/utils/celery_utils.py
View File

@@ -8,8 +8,6 @@ celery_app = Celery()
def init_celery(celery, app, is_beat=False): def init_celery(celery, app, is_beat=False):
celery_app.main = app.name celery_app.main = app.name
app.logger.debug(f'CELERY_BROKER_URL: {app.config["CELERY_BROKER_URL"]}')
app.logger.debug(f'CELERY_RESULT_BACKEND: {app.config["CELERY_RESULT_BACKEND"]}')
celery_config = { celery_config = {
'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'), 'broker_url': app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),

613
common/utils/config_field_types.py Normal file
View File

@@ -0,0 +1,613 @@
from typing import Optional, List, Union, Dict, Any, Pattern
from pydantic import BaseModel, field_validator, model_validator
from typing_extensions import Annotated
import re
from datetime import datetime
import json
from textwrap import dedent
import yaml
from dataclasses import dataclass
class TaggingField(BaseModel):
"""Represents a single tagging field configuration"""
type: str
required: bool = False
description: Optional[str] = None
allowed_values: Optional[List[Any]] = None # for enum type
min_value: Optional[Union[int, float]] = None # for numeric types
max_value: Optional[Union[int, float]] = None # for numeric types
@field_validator('type', mode='before')
@classmethod
def validate_type(cls, v: str) -> str:
valid_types = ['string', 'integer', 'float', 'date', 'enum']
if v not in valid_types:
raise ValueError(f'type must be one of {valid_types}')
return v
@model_validator(mode='after')
def validate_field_constraints(self) -> 'TaggingField':
# Validate enum constraints
if self.type == 'enum':
if not self.allowed_values:
raise ValueError('allowed_values must be provided for enum type')
elif self.allowed_values is not None:
raise ValueError('allowed_values only valid for enum type')
# Validate numeric constraints
if self.type not in ('integer', 'float'):
if self.min_value is not None or self.max_value is not None:
raise ValueError('min_value/max_value only valid for numeric types')
else:
if self.min_value is not None and self.max_value is not None and self.min_value >= self.max_value:
raise ValueError('min_value must be less than max_value')
return self
class TaggingFields(BaseModel):
"""Represents a collection of tagging fields, mapped by their names"""
fields: Dict[str, TaggingField]
@classmethod
def from_dict(cls, data: Dict[str, Dict[str, Any]]) -> 'TaggingFields':
return cls(fields={
field_name: TaggingField(**field_config)
for field_name, field_config in data.items()
})
def to_dict(self) -> Dict[str, Dict[str, Any]]:
return {
field_name: field.model_dump(exclude_none=True)
for field_name, field in self.fields.items()
}
class ArgumentConstraint(BaseModel):
"""Base class for all argument constraints"""
description: Optional[str] = None
error_message: Optional[str] = None
class NumericConstraint(ArgumentConstraint):
"""Constraints for numeric values (int/float)"""
min_value: Optional[float] = None
max_value: Optional[float] = None
include_min: bool = True # True for >= min_value, False for > min_value
include_max: bool = True # True for <= max_value, False for < max_value
@model_validator(mode='after')
def validate_ranges(self) -> 'NumericConstraint':
if self.min_value is not None and self.max_value is not None:
if self.min_value > self.max_value:
raise ValueError("min_value must be less than or equal to max_value")
return self
def validate(self, value: Union[int, float]) -> bool:
if self.min_value is not None:
if self.include_min and value < self.min_value:
return False
if not self.include_min and value <= self.min_value:
return False
if self.max_value is not None:
if self.include_max and value > self.max_value:
return False
if not self.include_max and value >= self.max_value:
return False
return True
class StringConstraint(ArgumentConstraint):
"""Constraints for string values"""
min_length: Optional[int] = None
max_length: Optional[int] = None
patterns: Optional[List[str]] = None # List of regex patterns to match
pattern_match_all: bool = False # If True, string must match all patterns
forbidden_patterns: Optional[List[str]] = None # List of regex patterns that must not match
allow_empty: bool = False
@field_validator('patterns', 'forbidden_patterns')
@classmethod
def validate_patterns(cls, v: Optional[List[str]]) -> Optional[List[str]]:
if v is not None:
# Validate each pattern compiles
for pattern in v:
try:
re.compile(pattern)
except re.error as e:
raise ValueError(f"Invalid regex pattern '{pattern}': {str(e)}")
return v
def validate(self, value: str) -> bool:
if not self.allow_empty and not value:
return False
if self.min_length is not None and len(value) < self.min_length:
return False
if self.max_length is not None and len(value) > self.max_length:
return False
if self.patterns:
matches = [bool(re.search(pattern, value)) for pattern in self.patterns]
if self.pattern_match_all and not all(matches):
return False
if not self.pattern_match_all and not any(matches):
return False
if self.forbidden_patterns:
for pattern in self.forbidden_patterns:
if re.search(pattern, value):
return False
return True
class DateConstraint(ArgumentConstraint):
"""Constraints for date values"""
min_date: Optional[datetime] = None
max_date: Optional[datetime] = None
include_min: bool = True
include_max: bool = True
allowed_formats: Optional[List[str]] = None # List of allowed date formats
@model_validator(mode='after')
def validate_ranges(self) -> 'DateConstraint':
if self.min_date and self.max_date and self.min_date > self.max_date:
raise ValueError("min_date must be less than or equal to max_date")
return self
def validate(self, value: datetime) -> bool:
if self.min_date is not None:
if self.include_min and value < self.min_date:
return False
if not self.include_min and value <= self.min_date:
return False
if self.max_date is not None:
if self.include_max and value > self.max_date:
return False
if not self.include_max and value >= self.max_date:
return False
return True
class EnumConstraint(ArgumentConstraint):
"""Constraints for enum values"""
allowed_values: List[Any]
case_sensitive: bool = True # For string enums
allow_multiple: bool = False # If True, value can be a list of allowed values
min_selections: Optional[int] = None # When allow_multiple is True
max_selections: Optional[int] = None # When allow_multiple is True
@model_validator(mode='after')
def validate_selections(self) -> 'EnumConstraint':
if self.allow_multiple:
if self.min_selections is not None and self.max_selections is not None:
if self.min_selections > self.max_selections:
raise ValueError("min_selections must be less than or equal to max_selections")
if self.max_selections > len(self.allowed_values):
raise ValueError("max_selections cannot be greater than number of allowed values")
return self
def validate(self, value: Union[Any, List[Any]]) -> bool:
if self.allow_multiple:
if not isinstance(value, list):
return False
if self.min_selections is not None and len(value) < self.min_selections:
return False
if self.max_selections is not None and len(value) > self.max_selections:
return False
for v in value:
if not self._validate_single_value(v):
return False
else:
return self._validate_single_value(value)
return True
def _validate_single_value(self, value: Any) -> bool:
if isinstance(value, str) and not self.case_sensitive:
return any(str(value).lower() == str(v).lower() for v in self.allowed_values)
return value in self.allowed_values
class ArgumentDefinition(BaseModel):
"""Defines an argument with its type and constraints"""
name: str
type: str
description: Optional[str] = None
required: bool = False
default: Optional[Any] = None
constraints: Optional[Union[NumericConstraint, StringConstraint, DateConstraint, EnumConstraint]] = None
@field_validator('type')
@classmethod
def validate_type(cls, v: str) -> str:
valid_types = ['string', 'integer', 'float', 'date', 'enum']
if v not in valid_types:
raise ValueError(f'type must be one of {valid_types}')
return v
@model_validator(mode='after')
def validate_constraints(self) -> 'ArgumentDefinition':
if self.constraints:
expected_constraint_types = {
'string': StringConstraint,
'integer': NumericConstraint,
'float': NumericConstraint,
'date': DateConstraint,
'enum': EnumConstraint
}
expected_type = expected_constraint_types.get(self.type)
if not isinstance(self.constraints, expected_type):
raise ValueError(f'Constraints for type {self.type} must be of type {expected_type.__name__}')
if self.default is not None:
if not self.constraints.validate(self.default):
raise ValueError(f'Default value does not satisfy constraints for {self.name}')
return self
class ArgumentDefinitions(BaseModel):
"""Collection of argument definitions"""
arguments: Dict[str, ArgumentDefinition]
@classmethod
def from_dict(cls, data: Dict[str, Dict[str, Any]]) -> 'ArgumentDefinitions':
return cls(arguments={
arg_name: ArgumentDefinition(**arg_config)
for arg_name, arg_config in data.items()
})
def to_dict(self) -> Dict[str, Dict[str, Any]]:
return {
arg_name: arg.model_dump(exclude_none=True)
for arg_name, arg in self.arguments.items()
}
def validate_argument_values(self, values: Dict[str, Any]) -> Dict[str, str]:
"""
Validate a set of argument values against their definitions
Returns a dictionary of error messages for invalid arguments
"""
errors = {}
# Check for required arguments
for name, arg_def in self.arguments.items():
if arg_def.required and name not in values:
errors[name] = "Required argument missing"
continue
if name in values:
value = values[name]
# Validate type
try:
if arg_def.type == 'integer':
value = int(value)
elif arg_def.type == 'float':
value = float(value)
elif arg_def.type == 'date' and isinstance(value, str):
if arg_def.constraints and arg_def.constraints.allowed_formats:
for fmt in arg_def.constraints.allowed_formats:
try:
value = datetime.strptime(value, fmt)
break
except ValueError:
continue
else:
errors[
name] = f"Invalid date format. Allowed formats: {arg_def.constraints.allowed_formats}"
continue
except (ValueError, TypeError):
errors[name] = f"Invalid type. Expected {arg_def.type}"
continue
# Validate constraints
if arg_def.constraints and not arg_def.constraints.validate(value):
errors[name] = arg_def.constraints.error_message or "Value does not satisfy constraints"
return errors
@dataclass
class DocumentationFormat:
"""Constants for documentation formats"""
MARKDOWN = "markdown"
JSON = "json"
YAML = "yaml"
@dataclass
class DocumentationVersion:
"""Constants for documentation versions"""
BASIC = "basic" # Original documentation without retriever info
EXTENDED = "extended" # Including retriever documentation
def _generate_argument_constraints(field_config: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Generate possible argument constraints based on field type"""
constraints = []
base_constraint = {
"description": f"Constraint for {field_config.get('description', 'field')}",
"error_message": "Optional custom error message"
}
if field_config["type"] == "integer" or field_config["type"] == "float":
constraints.append({
**base_constraint,
"type": "NumericConstraint",
"possible_constraints": {
"min_value": "number",
"max_value": "number",
"include_min": "boolean",
"include_max": "boolean"
},
"example": {
"min_value": field_config.get("min_value", 0),
"max_value": field_config.get("max_value", 100),
"include_min": True,
"include_max": True
}
})
elif field_config["type"] == "string":
constraints.append({
**base_constraint,
"type": "StringConstraint",
"possible_constraints": {
"min_length": "integer",
"max_length": "integer",
"patterns": "list[str]",
"pattern_match_all": "boolean",
"forbidden_patterns": "list[str]",
"allow_empty": "boolean"
},
"example": {
"min_length": 1,
"max_length": 100,
"patterns": ["^[A-Za-z0-9]+$"],
"pattern_match_all": False,
"forbidden_patterns": ["^test_", "_temp$"],
"allow_empty": False
}
})
elif field_config["type"] == "enum":
constraints.append({
**base_constraint,
"type": "EnumConstraint",
"possible_constraints": {
"allowed_values": f"list[{field_config.get('allowed_values', ['value1', 'value2'])}]",
"case_sensitive": "boolean",
"allow_multiple": "boolean",
"min_selections": "integer",
"max_selections": "integer"
},
"example": {
"allowed_values": field_config.get("allowed_values", ["value1", "value2"]),
"case_sensitive": True,
"allow_multiple": True,
"min_selections": 1,
"max_selections": 2
}
})
elif field_config["type"] == "date":
constraints.append({
**base_constraint,
"type": "DateConstraint",
"possible_constraints": {
"min_date": "datetime",
"max_date": "datetime",
"include_min": "boolean",
"include_max": "boolean",
"allowed_formats": "list[str]"
},
"example": {
"min_date": "2024-01-01T00:00:00",
"max_date": "2024-12-31T23:59:59",
"include_min": True,
"include_max": True,
"allowed_formats": ["%Y-%m-%d", "%Y/%m/%d"]
}
})
return constraints
def generate_field_documentation(
tagging_fields: Dict[str, Any],
format: str = "markdown",
version: str = "basic"
) -> str:
"""
Generate documentation for tagging fields configuration.
Args:
tagging_fields: Dictionary containing tagging fields configuration
format: Output format ("markdown", "json", or "yaml")
version: Documentation version ("basic" or "extended")
Returns:
str: Formatted documentation
"""
if version not in [DocumentationVersion.BASIC, DocumentationVersion.EXTENDED]:
raise ValueError(f"Unsupported documentation version: {version}")
# Normalize fields configuration
normalized_fields = {}
for field_name, field_config in tagging_fields.items():
field_doc = {
"name": field_name,
"type": field_config["type"],
"required": field_config.get("required", False),
"description": field_config.get("description", "No description provided"),
"constraints": []
}
# Only include possible arguments in extended version
if version == DocumentationVersion.EXTENDED:
field_doc["possible_arguments"] = _generate_argument_constraints(field_config)
# Add type-specific constraints
if field_config["type"] == "integer" or field_config["type"] == "float":
if "min_value" in field_config:
field_doc["constraints"].append(
f"Minimum value: {field_config['min_value']}")
if "max_value" in field_config:
field_doc["constraints"].append(
f"Maximum value: {field_config['max_value']}")
elif field_config["type"] == "string":
if "min_length" in field_config:
field_doc["constraints"].append(
f"Minimum length: {field_config['min_length']}")
if "max_length" in field_config:
field_doc["constraints"].append(
f"Maximum length: {field_config['max_length']}")
if "patterns" in field_config:
field_doc["constraints"].append(
f"Must match patterns: {', '.join(field_config['patterns'])}")
elif field_config["type"] == "enum":
if "allowed_values" in field_config:
field_doc["constraints"].append(
f"Allowed values: {', '.join(str(v) for v in field_config['allowed_values'])}")
elif field_config["type"] == "date":
if "min_date" in field_config:
field_doc["constraints"].append(
f"Minimum date: {field_config['min_date']}")
if "max_date" in field_config:
field_doc["constraints"].append(
f"Maximum date: {field_config['max_date']}")
if "allowed_formats" in field_config:
field_doc["constraints"].append(
f"Allowed formats: {', '.join(field_config['allowed_formats'])}")
normalized_fields[field_name] = field_doc
# Generate documentation in requested format
if format == DocumentationFormat.MARKDOWN:
return _generate_markdown_docs(normalized_fields, version)
elif format == DocumentationFormat.JSON:
return _generate_json_docs(normalized_fields, version)
elif format == DocumentationFormat.YAML:
return _generate_yaml_docs(normalized_fields, version)
else:
raise ValueError(f"Unsupported documentation format: {format}")
def _generate_markdown_docs(fields: Dict[str, Any], version: str) -> str:
"""Generate markdown documentation"""
docs = ["# Tagging Fields Documentation\n"]
# Add overview table
docs.append("## Fields Overview\n")
docs.append("| Field Name | Type | Required | Description |")
docs.append("|------------|------|----------|-------------|")
for field_name, field in fields.items():
docs.append(
f"| {field_name} | {field['type']} | "
f"{'Yes' if field['required'] else 'No'} | {field['description']} |"
)
# Add detailed field specifications
docs.append("\n## Detailed Field Specifications\n")
for field_name, field in fields.items():
docs.append(f"### {field_name}\n")
docs.append(f"**Type:** {field['type']}")
docs.append(f"**Required:** {'Yes' if field['required'] else 'No'}")
docs.append(f"**Description:** {field['description']}\n")
if field["constraints"]:
docs.append("**Field Constraints:**")
for constraint in field["constraints"]:
docs.append(f"- {constraint}")
docs.append("")
# Add retriever argument documentation only in extended version
if version == DocumentationVersion.EXTENDED and "possible_arguments" in field:
docs.append("**Possible Retriever Arguments:**")
for arg_constraint in field["possible_arguments"]:
docs.append(f"\n*{arg_constraint['type']}*")
docs.append(f"Description: {arg_constraint['description']}")
docs.append("\nPossible constraints:")
for const_name, const_type in arg_constraint["possible_constraints"].items():
docs.append(f"- `{const_name}`: {const_type}")
docs.append("\nExample:")
docs.append("```python")
docs.append(json.dumps(arg_constraint["example"], indent=2))
docs.append("```\n")
# Add example retriever configuration only in extended version
if version == DocumentationVersion.EXTENDED:
docs.append("\n## Example Retriever Configuration\n")
docs.append("```python")
example_config = {
"metadata_filters": {
field_name: field["possible_arguments"][0]["example"]
for field_name, field in fields.items()
if "possible_arguments" in field
}
}
docs.append(json.dumps(example_config, indent=2))
docs.append("```")
return "\n".join(docs)
def _generate_json_docs(fields: Dict[str, Any], version: str) -> str:
"""Generate JSON documentation"""
doc = {
"tagging_fields_documentation": {
"version": version,
"fields": fields
}
}
if version == DocumentationVersion.EXTENDED:
doc["tagging_fields_documentation"]["example_retriever_config"] = {
"metadata_filters": {
field_name: field["possible_arguments"][0]["example"]
for field_name, field in fields.items()
if "possible_arguments" in field
}
}
return json.dumps(doc, indent=2)
def _generate_yaml_docs(fields: Dict[str, Any], version: str) -> str:
"""Generate YAML documentation"""
doc = {
"tagging_fields_documentation": {
"version": version,
"fields": fields
}
}
if version == DocumentationVersion.EXTENDED:
doc["tagging_fields_documentation"]["example_retriever_config"] = {
"metadata_filters": {
field_name: field["possible_arguments"][0]["example"]
for field_name, field in fields.items()
if "possible_arguments" in field
}
}
return yaml.dump(doc, sort_keys=False, default_flow_style=False)

15
common/utils/cors_utils.py
View File

@@ -5,10 +5,8 @@ from common.models.user import Tenant, TenantDomain
def get_allowed_origins(tenant_id): def get_allowed_origins(tenant_id):
session_key = f"allowed_origins_{tenant_id}" session_key = f"allowed_origins_{tenant_id}"
if session_key in session: if session_key in session:
current_app.logger.debug(f"Fetching allowed origins for tenant {tenant_id} from session")
return session[session_key] return session[session_key]
current_app.logger.debug(f"Fetching allowed origins for tenant {tenant_id} from database")
tenant_domains = TenantDomain.query.filter_by(tenant_id=int(tenant_id)).all() tenant_domains = TenantDomain.query.filter_by(tenant_id=int(tenant_id)).all()
allowed_origins = [domain.domain for domain in tenant_domains] allowed_origins = [domain.domain for domain in tenant_domains]
@@ -18,14 +16,8 @@ def get_allowed_origins(tenant_id):
def cors_after_request(response, prefix): def cors_after_request(response, prefix):
current_app.logger.debug(f'CORS after request: {request.path}, prefix: {prefix}')
current_app.logger.debug(f'request.headers: {request.headers}')
current_app.logger.debug(f'request.args: {request.args}')
current_app.logger.debug(f'request is json?: {request.is_json}')
# Exclude health checks from checks # Exclude health checks from checks
if request.path.startswith('/healthz') or request.path.startswith('/_healthz'): if request.path.startswith('/healthz') or request.path.startswith('/_healthz'):
current_app.logger.debug('Skipping CORS headers for health checks')
response.headers.add('Access-Control-Allow-Origin', '*') response.headers.add('Access-Control-Allow-Origin', '*')
response.headers.add('Access-Control-Allow-Headers', '*') response.headers.add('Access-Control-Allow-Headers', '*')
response.headers.add('Access-Control-Allow-Methods', '*') response.headers.add('Access-Control-Allow-Methods', '*')
@@ -36,7 +28,6 @@ def cors_after_request(response, prefix):
# Try to get tenant_id from JSON payload # Try to get tenant_id from JSON payload
json_data = request.get_json(silent=True) json_data = request.get_json(silent=True)
current_app.logger.debug(f'request.get_json(silent=True): {json_data}')
if json_data and 'tenant_id' in json_data: if json_data and 'tenant_id' in json_data:
tenant_id = json_data['tenant_id'] tenant_id = json_data['tenant_id']
@@ -44,23 +35,17 @@ def cors_after_request(response, prefix):
# Fallback to get tenant_id from query parameters or headers if JSON is not available # Fallback to get tenant_id from query parameters or headers if JSON is not available
tenant_id = request.args.get('tenant_id') or request.args.get('tenantId') or request.headers.get('X-Tenant-ID') tenant_id = request.args.get('tenant_id') or request.args.get('tenantId') or request.headers.get('X-Tenant-ID')
current_app.logger.debug(f'Identified tenant_id: {tenant_id}')
if tenant_id: if tenant_id:
allowed_origins = get_allowed_origins(tenant_id) allowed_origins = get_allowed_origins(tenant_id)
current_app.logger.debug(f'Allowed origins for tenant {tenant_id}: {allowed_origins}')
else: else:
current_app.logger.warning('tenant_id not found in request') current_app.logger.warning('tenant_id not found in request')
origin = request.headers.get('Origin') origin = request.headers.get('Origin')
current_app.logger.debug(f'Origin: {origin}')
if origin in allowed_origins: if origin in allowed_origins:
response.headers.add('Access-Control-Allow-Origin', origin) response.headers.add('Access-Control-Allow-Origin', origin)
response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization') response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
response.headers.add('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS') response.headers.add('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS')
response.headers.add('Access-Control-Allow-Credentials', 'true') response.headers.add('Access-Control-Allow-Credentials', 'true')
current_app.logger.debug(f'CORS headers set for origin: {origin}')
else: else:
current_app.logger.warning(f'Origin {origin} not allowed') current_app.logger.warning(f'Origin {origin} not allowed')

3
common/utils/debug_utils.py
View File

@@ -36,7 +36,7 @@ def log_request_middleware(app):
@app.before_request @app.before_request
def log_session_state_before(): def log_session_state_before():
app.logger.debug(f'Session state before request: {session.items()}') pass
# @app.after_request # @app.after_request
# def log_response_info(response): # def log_response_info(response):
@@ -58,5 +58,4 @@ def log_request_middleware(app):
@app.after_request @app.after_request
def log_session_state_after(response): def log_session_state_after(response):
app.logger.debug(f'Session state after request: {session.items()}')
return response return response

8
common/utils/document_utils.py
View File

@@ -24,6 +24,7 @@ def create_document_stack(api_input, file, filename, extension, tenant_id):
# Create the DocumentVersion # Create the DocumentVersion
new_doc_vers = create_version_for_document(new_doc, tenant_id, new_doc_vers = create_version_for_document(new_doc, tenant_id,
api_input.get('url', ''), api_input.get('url', ''),
api_input.get('sub_file_type', ''),
api_input.get('language', 'en'), api_input.get('language', 'en'),
api_input.get('user_context', ''), api_input.get('user_context', ''),
api_input.get('user_metadata'), api_input.get('user_metadata'),
@@ -64,7 +65,7 @@ def create_document(form, filename, catalog_id):
return new_doc return new_doc
def create_version_for_document(document, tenant_id, url, language, user_context, user_metadata, catalog_properties): def create_version_for_document(document, tenant_id, url, sub_file_type, language, user_context, user_metadata, catalog_properties):
new_doc_vers = DocumentVersion() new_doc_vers = DocumentVersion()
if url != '': if url != '':
new_doc_vers.url = url new_doc_vers.url = url
@@ -83,6 +84,9 @@ def create_version_for_document(document, tenant_id, url, language, user_context
if catalog_properties != '' and catalog_properties is not None: if catalog_properties != '' and catalog_properties is not None:
new_doc_vers.catalog_properties = catalog_properties new_doc_vers.catalog_properties = catalog_properties
if sub_file_type != '':
new_doc_vers.sub_file_type = sub_file_type
new_doc_vers.document = document new_doc_vers.document = document
set_logging_information(new_doc_vers, dt.now(tz.utc)) set_logging_information(new_doc_vers, dt.now(tz.utc))
@@ -237,8 +241,6 @@ def start_embedding_task(tenant_id, doc_vers_id):
def validate_file_type(extension): def validate_file_type(extension):
current_app.logger.debug(f'Validating file type {extension}')
current_app.logger.debug(f'Supported file types: {current_app.config["SUPPORTED_FILE_TYPES"]}')
if extension not in current_app.config['SUPPORTED_FILE_TYPES']: if extension not in current_app.config['SUPPORTED_FILE_TYPES']:
raise EveAIUnsupportedFileType(f"Filetype {extension} is currently not supported. " raise EveAIUnsupportedFileType(f"Filetype {extension} is currently not supported. "
f"Supported filetypes: {', '.join(current_app.config['SUPPORTED_FILE_TYPES'])}") f"Supported filetypes: {', '.join(current_app.config['SUPPORTED_FILE_TYPES'])}")

7
common/utils/eveai_exceptions.py
View File

@@ -10,6 +10,7 @@ class EveAIException(Exception):
def to_dict(self): def to_dict(self):
rv = dict(self.payload or ()) rv = dict(self.payload or ())
rv['message'] = self.message rv['message'] = self.message
rv['error'] = self.__class__.__name__
return rv return rv
@@ -41,3 +42,9 @@ class EveAINoLicenseForTenant(EveAIException):
super().__init__(message, status_code, payload) super().__init__(message, status_code, payload)
class EveAITenantNotFound(EveAIException):
"""Raised when a tenant is not found"""
def __init__(self, message="Tenant not found", status_code=400, payload=None):
super().__init__(message, status_code, payload)

3
common/utils/middleware.py
View File

@@ -24,9 +24,6 @@ def mw_before_request():
if not tenant_id: if not tenant_id:
raise Exception('Cannot switch schema for tenant: no tenant defined in session') raise Exception('Cannot switch schema for tenant: no tenant defined in session')
for role in current_user.roles:
current_app.logger.debug(f'In middleware: User {current_user.email} has role {role.name}')
# user = User.query.get(current_user.id) # user = User.query.get(current_user.id)
if current_user.has_role('Super User') or current_user.tenant_id == tenant_id: if current_user.has_role('Super User') or current_user.tenant_id == tenant_id:
Database(tenant_id).switch_schema() Database(tenant_id).switch_schema()

503
common/utils/model_utils.py
View File

@@ -1,249 +1,36 @@
import os import os
from typing import Dict, Any, Optional
import langcodes import langcodes
from flask import current_app
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain_anthropic import ChatAnthropic
from langchain_core.pydantic_v1 import BaseModel, Field
from typing import List, Any, Iterator
from collections.abc import MutableMapping
from openai import OpenAI
from portkey_ai import createHeaders, PORTKEY_GATEWAY_URL
from portkey_ai.langchain.portkey_langchain_callback_handler import LangchainCallbackHandler
from common.langchain.llm_metrics_handler import LLMMetricsHandler from common.langchain.llm_metrics_handler import LLMMetricsHandler
from common.langchain.templates.template_manager import TemplateManager
from langchain_openai import OpenAIEmbeddings, ChatOpenAI, OpenAI
from langchain_anthropic import ChatAnthropic
from flask import current_app
from datetime import datetime as dt, timezone as tz
from common.langchain.tracked_openai_embeddings import TrackedOpenAIEmbeddings from common.langchain.tracked_openai_embeddings import TrackedOpenAIEmbeddings
from common.langchain.tracked_transcribe import tracked_transcribe from common.langchain.tracked_transcription import TrackedOpenAITranscription
from common.models.document import EmbeddingSmallOpenAI, EmbeddingLargeOpenAI, Catalog
from common.models.user import Tenant from common.models.user import Tenant
from common.utils.cache.base import CacheHandler
from config.model_config import MODEL_CONFIG from config.model_config import MODEL_CONFIG
from common.utils.business_event_context import current_event from common.extensions import template_manager, cache_manager
from common.models.document import EmbeddingLargeOpenAI, EmbeddingSmallOpenAI
from common.utils.eveai_exceptions import EveAITenantNotFound
class CitedAnswer(BaseModel): def create_language_template(template: str, language: str) -> str:
"""Default docstring - to be replaced with actual prompt""" """
Replace language placeholder in template with specified language
answer: str = Field( Args:
..., template: Template string with {language} placeholder
description="The answer to the user question, based on the given sources", language: Language code to insert
)
citations: List[int] = Field(
...,
description="The integer IDs of the SPECIFIC sources that were used to generate the answer"
)
insufficient_info: bool = Field(
False, # Default value is set to False
description="A boolean indicating wether given sources were sufficient or not to generate the answer"
)
Returns:
def set_language_prompt_template(cls, language_prompt): str: Template with language placeholder replaced
cls.__doc__ = language_prompt """
class ModelVariables(MutableMapping):
def __init__(self, tenant: Tenant, catalog_id=None):
self.tenant = tenant
self.catalog_id = catalog_id
self._variables = self._initialize_variables()
self._embedding_model = None
self._llm = None
self._llm_no_rag = None
self._transcription_client = None
self._prompt_templates = {}
self._embedding_db_model = None
self.llm_metrics_handler = LLMMetricsHandler()
self._transcription_client = None
def _initialize_variables(self):
variables = {}
# Get the Catalog if catalog_id is passed
if self.catalog_id:
catalog = Catalog.query.get_or_404(self.catalog_id)
# We initialize the variables that are available knowing the tenant.
variables['embed_tuning'] = catalog.embed_tuning or False
# Set HTML Chunking Variables
variables['html_tags'] = catalog.html_tags
variables['html_end_tags'] = catalog.html_end_tags
variables['html_included_elements'] = catalog.html_included_elements
variables['html_excluded_elements'] = catalog.html_excluded_elements
variables['html_excluded_classes'] = catalog.html_excluded_classes
# Set Chunk Size variables
variables['min_chunk_size'] = catalog.min_chunk_size
variables['max_chunk_size'] = catalog.max_chunk_size
# Set the RAG Context (will have to change once specialists are defined
variables['rag_context'] = self.tenant.rag_context or " "
# Temporary setting until we have Specialists
variables['rag_tuning'] = False
variables['RAG_temperature'] = 0.3
variables['no_RAG_temperature'] = 0.5
variables['k'] = 8
variables['similarity_threshold'] = 0.4
# Set model providers
variables['embedding_provider'], variables['embedding_model'] = self.tenant.embedding_model.rsplit('.', 1)
variables['llm_provider'], variables['llm_model'] = self.tenant.llm_model.rsplit('.', 1)
variables["templates"] = current_app.config['PROMPT_TEMPLATES'][(f"{variables['llm_provider']}."
f"{variables['llm_model']}")]
current_app.logger.info(f"Loaded prompt templates: \n")
current_app.logger.info(f"{variables['templates']}")
# Set model-specific configurations
model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {})
variables.update(model_config)
variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][self.tenant.llm_model]
if variables['tool_calling_supported']:
variables['cited_answer_cls'] = CitedAnswer
variables['max_compression_duration'] = current_app.config['MAX_COMPRESSION_DURATION']
variables['max_transcription_duration'] = current_app.config['MAX_TRANSCRIPTION_DURATION']
variables['compression_cpu_limit'] = current_app.config['COMPRESSION_CPU_LIMIT']
variables['compression_process_delay'] = current_app.config['COMPRESSION_PROCESS_DELAY']
return variables
@property
def embedding_model(self):
api_key = os.getenv('OPENAI_API_KEY')
model = self._variables['embedding_model']
self._embedding_model = TrackedOpenAIEmbeddings(api_key=api_key,
model=model,
)
self._embedding_db_model = EmbeddingSmallOpenAI \
if model == 'text-embedding-3-small' \
else EmbeddingLargeOpenAI
return self._embedding_model
@property
def llm(self):
api_key = self.get_api_key_for_llm()
self._llm = ChatOpenAI(api_key=api_key,
model=self._variables['llm_model'],
temperature=self._variables['RAG_temperature'],
callbacks=[self.llm_metrics_handler])
return self._llm
@property
def llm_no_rag(self):
api_key = self.get_api_key_for_llm()
self._llm_no_rag = ChatOpenAI(api_key=api_key,
model=self._variables['llm_model'],
temperature=self._variables['RAG_temperature'],
callbacks=[self.llm_metrics_handler])
return self._llm_no_rag
def get_api_key_for_llm(self):
if self._variables['llm_provider'] == 'openai':
api_key = os.getenv('OPENAI_API_KEY')
else: # self._variables['llm_provider'] == 'anthropic'
api_key = os.getenv('ANTHROPIC_API_KEY')
return api_key
@property
def transcription_client(self):
api_key = os.getenv('OPENAI_API_KEY')
self._transcription_client = OpenAI(api_key=api_key, )
self._variables['transcription_model'] = 'whisper-1'
return self._transcription_client
def transcribe(self, *args, **kwargs):
return tracked_transcribe(self._transcription_client, *args, **kwargs)
@property
def embedding_db_model(self):
if self._embedding_db_model is None:
self._embedding_db_model = self.get_embedding_db_model()
return self._embedding_db_model
def get_embedding_db_model(self):
current_app.logger.debug("In get_embedding_db_model")
if self._embedding_db_model is None:
self._embedding_db_model = EmbeddingSmallOpenAI \
if self._variables['embedding_model'] == 'text-embedding-3-small' \
else EmbeddingLargeOpenAI
current_app.logger.debug(f"Embedding DB Model: {self._embedding_db_model}")
return self._embedding_db_model
def get_prompt_template(self, template_name: str) -> str:
current_app.logger.info(f"Getting prompt template for {template_name}")
if template_name not in self._prompt_templates:
self._prompt_templates[template_name] = self._load_prompt_template(template_name)
return self._prompt_templates[template_name]
def _load_prompt_template(self, template_name: str) -> str:
# In the future, this method will make an API call to Portkey
# For now, we'll simulate it with a placeholder implementation
# You can replace this with your current prompt loading logic
return self._variables['templates'][template_name]
def __getitem__(self, key: str) -> Any:
current_app.logger.debug(f"ModelVariables: Getting {key}")
# Support older template names (suffix = _template)
if key.endswith('_template'):
key = key[:-len('_template')]
current_app.logger.debug(f"ModelVariables: Getting modified {key}")
if key == 'embedding_model':
return self.embedding_model
elif key == 'embedding_db_model':
return self.embedding_db_model
elif key == 'llm':
return self.llm
elif key == 'llm_no_rag':
return self.llm_no_rag
elif key == 'transcription_client':
return self.transcription_client
elif key in self._variables.get('prompt_templates', []):
return self.get_prompt_template(key)
else:
value = self._variables.get(key)
if value is not None:
return value
else:
raise KeyError(f'Variable {key} does not exist in ModelVariables')
def __setitem__(self, key: str, value: Any) -> None:
self._variables[key] = value
def __delitem__(self, key: str) -> None:
del self._variables[key]
def __iter__(self) -> Iterator[str]:
return iter(self._variables)
def __len__(self):
return len(self._variables)
def get(self, key: str, default: Any = None) -> Any:
return self.__getitem__(key) or default
def update(self, **kwargs) -> None:
self._variables.update(kwargs)
def items(self):
return self._variables.items()
def keys(self):
return self._variables.keys()
def values(self):
return self._variables.values()
def select_model_variables(tenant, catalog_id=None):
model_variables = ModelVariables(tenant=tenant, catalog_id=catalog_id)
return model_variables
def create_language_template(template, language):
try: try:
full_language = langcodes.Language.make(language=language) full_language = langcodes.Language.make(language=language)
language_template = template.replace('{language}', full_language.display_name()) language_template = template.replace('{language}', full_language.display_name())
@@ -253,5 +40,249 @@ def create_language_template(template, language):
return language_template return language_template
def replace_variable_in_template(template, variable, value): def replace_variable_in_template(template: str, variable: str, value: str) -> str:
return template.replace(variable, value) """
Replace a variable placeholder in template with specified value
Args:
template: Template string with variable placeholder
variable: Variable placeholder to replace (e.g. "{tenant_context}")
value: Value to insert
Returns:
str: Template with variable placeholder replaced
"""
return template.replace(variable, value or "")
class ModelVariables:
"""Manages model-related variables and configurations"""
def __init__(self, tenant_id: int, variables: Dict[str, Any] = None):
"""
Initialize ModelVariables with tenant and optional template manager
Args:
tenant: Tenant instance
template_manager: Optional TemplateManager instance
"""
current_app.logger.info(f'Model variables initialized with tenant {tenant_id} and variables \n{variables}')
self.tenant_id = tenant_id
self._variables = variables if variables is not None else self._initialize_variables()
current_app.logger.info(f'Model _variables initialized to {self._variables}')
self._embedding_model = None
self._embedding_model_class = None
self._llm_instances = {}
self.llm_metrics_handler = LLMMetricsHandler()
self._transcription_model = None
def _initialize_variables(self) -> Dict[str, Any]:
"""Initialize the variables dictionary"""
variables = {}
tenant = Tenant.query.get(self.tenant_id)
if not tenant:
raise EveAITenantNotFound(f"Tenant {self.tenant_id} not found")
# Set model providers
variables['embedding_provider'], variables['embedding_model'] = tenant.embedding_model.split('.')
variables['llm_provider'], variables['llm_model'] = tenant.llm_model.split('.')
variables['llm_full_model'] = tenant.llm_model
# Set model-specific configurations
model_config = MODEL_CONFIG.get(variables['llm_provider'], {}).get(variables['llm_model'], {})
variables.update(model_config)
# Additional configurations
variables['annotation_chunk_length'] = current_app.config['ANNOTATION_TEXT_CHUNK_LENGTH'][tenant.llm_model]
variables['max_compression_duration'] = current_app.config['MAX_COMPRESSION_DURATION']
variables['max_transcription_duration'] = current_app.config['MAX_TRANSCRIPTION_DURATION']
variables['compression_cpu_limit'] = current_app.config['COMPRESSION_CPU_LIMIT']
variables['compression_process_delay'] = current_app.config['COMPRESSION_PROCESS_DELAY']
return variables
@property
def embedding_model(self):
"""Get the embedding model instance"""
if self._embedding_model is None:
api_key = os.getenv('OPENAI_API_KEY')
self._embedding_model = TrackedOpenAIEmbeddings(
api_key=api_key,
model=self._variables['embedding_model']
)
return self._embedding_model
@property
def embedding_model_class(self):
"""Get the embedding model class"""
if self._embedding_model_class is None:
if self._variables['embedding_model'] == 'text-embedding-3-large':
self._embedding_model_class = EmbeddingLargeOpenAI
else: # text-embedding-3-small
self._embedding_model_class = EmbeddingSmallOpenAI
return self._embedding_model_class
@property
def annotation_chunk_length(self):
return self._variables['annotation_chunk_length']
@property
def max_compression_duration(self):
return self._variables['max_compression_duration']
@property
def max_transcription_duration(self):
return self._variables['max_transcription_duration']
@property
def compression_cpu_limit(self):
return self._variables['compression_cpu_limit']
@property
def compression_process_delay(self):
return self._variables['compression_process_delay']
def get_llm(self, temperature: float = 0.3, **kwargs) -> Any:
"""
Get an LLM instance with specific configuration
Args:
temperature: The temperature for the LLM
**kwargs: Additional configuration parameters
Returns:
An instance of the configured LLM
"""
cache_key = f"{temperature}_{hash(frozenset(kwargs.items()))}"
if cache_key not in self._llm_instances:
provider = self._variables['llm_provider']
model = self._variables['llm_model']
if provider == 'openai':
self._llm_instances[cache_key] = ChatOpenAI(
api_key=os.getenv('OPENAI_API_KEY'),
model=model,
temperature=temperature,
callbacks=[self.llm_metrics_handler],
**kwargs
)
elif provider == 'anthropic':
self._llm_instances[cache_key] = ChatAnthropic(
api_key=os.getenv('ANTHROPIC_API_KEY'),
model=current_app.config['ANTHROPIC_LLM_VERSIONS'][model],
temperature=temperature,
callbacks=[self.llm_metrics_handler],
**kwargs
)
else:
raise ValueError(f"Unsupported LLM provider: {provider}")
return self._llm_instances[cache_key]
@property
def transcription_model(self) -> TrackedOpenAITranscription:
"""Get the transcription model instance"""
if self._transcription_model is None:
api_key = os.getenv('OPENAI_API_KEY')
self._transcription_model = TrackedOpenAITranscription(
api_key=api_key,
model='whisper-1'
)
return self._transcription_model
# Remove the old transcription-related methods since they're now handled by TrackedOpenAITranscription
@property
def transcription_client(self):
raise DeprecationWarning("Use transcription_model instead")
def transcribe(self, *args, **kwargs):
raise DeprecationWarning("Use transcription_model.transcribe() instead")
def get_template(self, template_name: str, version: Optional[str] = None) -> str:
"""
Get a template for the tenant's configured LLM
Args:
template_name: Name of the template to retrieve
version: Optional specific version to retrieve
Returns:
The template content
"""
try:
template = template_manager.get_template(
self._variables['llm_full_model'],
template_name,
version
)
return template.content
except Exception as e:
current_app.logger.error(f"Error getting template {template_name}: {str(e)}")
# Fall back to old template loading if template_manager fails
if template_name in self._variables.get('templates', {}):
return self._variables['templates'][template_name]
raise
class ModelVariablesCacheHandler(CacheHandler[ModelVariables]):
handler_name = 'model_vars_cache' # Used to access handler instance from cache_manager
def __init__(self, region):
super().__init__(region, 'model_variables')
self.configure_keys('tenant_id')
self.subscribe_to_model('Tenant', ['tenant_id'])
def to_cache_data(self, instance: ModelVariables) -> Dict[str, Any]:
return {
'tenant_id': instance.tenant_id,
'variables': instance._variables,
'last_updated': dt.now(tz=tz.utc).isoformat()
}
def from_cache_data(self, data: Dict[str, Any], tenant_id: int, **kwargs) -> ModelVariables:
instance = ModelVariables(tenant_id, data.get('variables'))
return instance
def should_cache(self, value: Dict[str, Any]) -> bool:
required_fields = {'tenant_id', 'variables'}
return all(field in value for field in required_fields)
# Register the handler with the cache manager
cache_manager.register_handler(ModelVariablesCacheHandler, 'model')
# Helper function to get cached model variables
def get_model_variables(tenant_id: int) -> ModelVariables:
return cache_manager.model_vars_cache.get(
lambda tenant_id: ModelVariables(tenant_id), # function to create ModelVariables if required
tenant_id=tenant_id
)
# Written in a long format, without lambda
# def get_model_variables(tenant_id: int) -> ModelVariables:
# """
# Get ModelVariables instance, either from cache or newly created
#
# Args:
# tenant_id: The tenant's ID
#
# Returns:
# ModelVariables: Instance with either cached or fresh data
#
# Raises:
# TenantNotFoundError: If tenant doesn't exist
# CacheStateError: If cached data is invalid
# """
#
# def create_new_instance(tenant_id: int) -> ModelVariables:
# """Creator function that's called when cache miss occurs"""
# return ModelVariables(tenant_id) # This will initialize fresh variables
#
# return cache_manager.model_vars_cache.get(
# create_new_instance, # Function to create new instance if needed
# tenant_id=tenant_id # Parameters passed to both get() and create_new_instance
# )

16
common/utils/os_utils.py
View File

@@ -1,4 +1,6 @@
import os import os
import sys
import gevent import gevent
import time import time
from flask import current_app from flask import current_app
@@ -28,3 +30,17 @@ def sync_folder(file_path):
dir_fd = os.open(file_path, os.O_RDONLY) dir_fd = os.open(file_path, os.O_RDONLY)
os.fsync(dir_fd) os.fsync(dir_fd)
os.close(dir_fd) os.close(dir_fd)
def get_project_root():
"""Get the root directory of the project."""
# Use the module that's actually running (not this file)
module = sys.modules['__main__']
if hasattr(module, '__file__'):
# Get the path to the main module
main_path = os.path.abspath(module.__file__)
# Get the root directory (where the main module is located)
return os.path.dirname(main_path)
else:
# Fallback: use current working directory
return os.getcwd()

1
common/utils/security.py
View File

@@ -4,7 +4,6 @@ from common.models.user import Tenant
# Definition of Trigger Handlers # Definition of Trigger Handlers
def set_tenant_session_data(sender, user, **kwargs): def set_tenant_session_data(sender, user, **kwargs):
current_app.logger.debug(f"Setting tenant session data for user {user.id}")
tenant = Tenant.query.filter_by(id=user.tenant_id).first() tenant = Tenant.query.filter_by(id=user.tenant_id).first()
session['tenant'] = tenant.to_dict() session['tenant'] = tenant.to_dict()
session['default_language'] = tenant.default_language session['default_language'] = tenant.default_language

7
common/utils/security_utils.py
View File

@@ -11,7 +11,7 @@ def confirm_token(token, expiration=3600):
try: try:
email = serializer.loads(token, salt=current_app.config['SECURITY_PASSWORD_SALT'], max_age=expiration) email = serializer.loads(token, salt=current_app.config['SECURITY_PASSWORD_SALT'], max_age=expiration)
except Exception as e: except Exception as e:
current_app.logger.debug(f'Error confirming token: {e}') current_app.logger.error(f'Error confirming token: {e}')
raise raise
return email return email
@@ -35,14 +35,11 @@ def generate_confirmation_token(email):
def send_confirmation_email(user): def send_confirmation_email(user):
current_app.logger.debug(f'Sending confirmation email to {user.email}')
if not test_smtp_connection(): if not test_smtp_connection():
raise Exception("Failed to connect to SMTP server") raise Exception("Failed to connect to SMTP server")
token = generate_confirmation_token(user.email) token = generate_confirmation_token(user.email)
confirm_url = prefixed_url_for('security_bp.confirm_email', token=token, _external=True) confirm_url = prefixed_url_for('security_bp.confirm_email', token=token, _external=True)
current_app.logger.debug(f'Confirmation URL: {confirm_url}')
html = render_template('email/activate.html', confirm_url=confirm_url) html = render_template('email/activate.html', confirm_url=confirm_url)
subject = "Please confirm your email" subject = "Please confirm your email"
@@ -56,10 +53,8 @@ def send_confirmation_email(user):
def send_reset_email(user): def send_reset_email(user):
current_app.logger.debug(f'Sending reset email to {user.email}')
token = generate_reset_token(user.email) token = generate_reset_token(user.email)
reset_url = prefixed_url_for('security_bp.reset_password', token=token, _external=True) reset_url = prefixed_url_for('security_bp.reset_password', token=token, _external=True)
current_app.logger.debug(f'Reset URL: {reset_url}')
html = render_template('email/reset_password.html', reset_url=reset_url) html = render_template('email/reset_password.html', reset_url=reset_url)
subject = "Reset Your Password" subject = "Reset Your Password"

112
common/utils/string_list_converter.py Normal file
View File

@@ -0,0 +1,112 @@
from typing import List, Union
import re
class StringListConverter:
"""Utility class for converting between comma-separated strings and lists"""
@staticmethod
def string_to_list(input_string: Union[str, None], allow_empty: bool = True) -> List[str]:
"""
Convert a comma-separated string to a list of strings.
Args:
input_string: Comma-separated string to convert
allow_empty: If True, returns empty list for None/empty input
If False, raises ValueError for None/empty input
Returns:
List of stripped strings
Raises:
ValueError: If input is None/empty and allow_empty is False
"""
if not input_string:
if allow_empty:
return []
raise ValueError("Input string cannot be None or empty")
return [item.strip() for item in input_string.split(',') if item.strip()]
@staticmethod
def list_to_string(input_list: Union[List[str], None], allow_empty: bool = True) -> str:
"""
Convert a list of strings to a comma-separated string.
Args:
input_list: List of strings to convert
allow_empty: If True, returns empty string for None/empty input
If False, raises ValueError for None/empty input
Returns:
Comma-separated string
Raises:
ValueError: If input is None/empty and allow_empty is False
"""
if not input_list:
if allow_empty:
return ''
raise ValueError("Input list cannot be None or empty")
return ', '.join(str(item).strip() for item in input_list)
@staticmethod
def validate_format(input_string: str,
allowed_chars: str = r'a-zA-Z0-9_\-',
min_length: int = 1,
max_length: int = 50) -> bool:
"""
Validate the format of items in a comma-separated string.
Args:
input_string: String to validate
allowed_chars: String of allowed characters (for regex pattern)
min_length: Minimum length for each item
max_length: Maximum length for each item
Returns:
bool: True if format is valid, False otherwise
"""
if not input_string:
return False
# Create regex pattern for individual items
pattern = f'^[{allowed_chars}]{{{min_length},{max_length}}}$'
try:
# Convert to list and check each item
items = StringListConverter.string_to_list(input_string)
return all(bool(re.match(pattern, item)) for item in items)
except Exception:
return False
@staticmethod
def validate_and_convert(input_string: str,
allowed_chars: str = r'a-zA-Z0-9_\-',
min_length: int = 1,
max_length: int = 50) -> List[str]:
"""
Validate and convert a comma-separated string to a list.
Args:
input_string: String to validate and convert
allowed_chars: String of allowed characters (for regex pattern)
min_length: Minimum length for each item
max_length: Maximum length for each item
Returns:
List of validated and converted strings
Raises:
ValueError: If input string format is invalid
"""
if not StringListConverter.validate_format(
input_string, allowed_chars, min_length, max_length
):
raise ValueError(
f"Invalid format. Items must be {min_length}-{max_length} characters "
f"long and contain only these characters: {allowed_chars}"
)
return StringListConverter.string_to_list(input_string)

2
common/utils/view_assistants.py
View File

@@ -44,7 +44,7 @@ def form_validation_failed(request, form):
for fieldName, errorMessages in form.errors.items(): for fieldName, errorMessages in form.errors.items():
for err in errorMessages: for err in errorMessages:
flash(f"Error in {fieldName}: {err}", 'danger') flash(f"Error in {fieldName}: {err}", 'danger')
current_app.logger.debug(f"Error in {fieldName}: {err}") current_app.logger.error(f"Error in {fieldName}: {err}")
def form_to_dict(form): def form_to_dict(form):

28
config/catalog_types.py
View File

@@ -3,7 +3,8 @@ CATALOG_TYPES = {
"STANDARD_CATALOG": { "STANDARD_CATALOG": {
"name": "Standard Catalog", "name": "Standard Catalog",
"Description": "A Catalog with information in Evie's Library, to be considered as a whole", "Description": "A Catalog with information in Evie's Library, to be considered as a whole",
"configuration": {} "configuration": {},
"document_version_configurations": []
}, },
"DOSSIER": { "DOSSIER": {
"name": "Dossier Catalog", "name": "Dossier Catalog",
@@ -21,31 +22,6 @@ CATALOG_TYPES = {
- min_value/max_value: range limits (for numeric types only)""", - min_value/max_value: range limits (for numeric types only)""",
"required": True, "required": True,
"default": {}, "default": {},
"field_properties": {
"type": {
"allowed_values": ["string", "integer", "float", "date", "enum"],
"required": True
},
"required": {
"type": "boolean",
"default": False
},
"description": {
"type": "string"
},
"allowed_values": {
"type": "list",
"description": "For enum type fields only"
},
"min_value": {
"type": "number",
"description": "For numeric fields only"
},
"max_value": {
"type": "number",
"description": "For numeric fields only"
}
}
} }
}, },
"document_version_configurations": ["tagging_fields"] "document_version_configurations": ["tagging_fields"]

28
config/config.py
View File

@@ -68,9 +68,6 @@ class Config(object):
ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', } ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', }
# Load prompt templates dynamically
PROMPT_TEMPLATES = {model: load_prompt_templates(model) for model in SUPPORTED_LLMS}
# Annotation text chunk length # Annotation text chunk length
ANNOTATION_TEXT_CHUNK_LENGTH = { ANNOTATION_TEXT_CHUNK_LENGTH = {
'openai.gpt-4o': 10000, 'openai.gpt-4o': 10000,
@@ -87,9 +84,6 @@ class Config(object):
# Anthropic API Keys # Anthropic API Keys
ANTHROPIC_API_KEY = environ.get('ANTHROPIC_API_KEY') ANTHROPIC_API_KEY = environ.get('ANTHROPIC_API_KEY')
# Portkey API Keys
PORTKEY_API_KEY = environ.get('PORTKEY_API_KEY')
# Celery settings # Celery settings
CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json'
@@ -181,13 +175,21 @@ class DevConfig(Config):
# file upload settings # file upload settings
# UPLOAD_FOLDER = '/app/tenant_files' # UPLOAD_FOLDER = '/app/tenant_files'
# Redis Settings
REDIS_URL = 'redis'
REDIS_PORT = '6379'
REDIS_BASE_URI = f'redis://{REDIS_URL}:{REDIS_PORT}'
# Celery settings # Celery settings
# eveai_app Redis Settings # eveai_app Redis Settings
CELERY_BROKER_URL = 'redis://redis:6379/0' CELERY_BROKER_URL = f'{REDIS_BASE_URI}/0'
CELERY_RESULT_BACKEND = 'redis://redis:6379/0' CELERY_RESULT_BACKEND = f'{REDIS_BASE_URI}/0'
# eveai_chat Redis Settings # eveai_chat Redis Settings
CELERY_BROKER_URL_CHAT = 'redis://redis:6379/3' CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
CELERY_RESULT_BACKEND_CHAT = 'redis://redis:6379/3' CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
# eveai_chat_workers cache Redis Settings
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
# Unstructured settings # Unstructured settings
# UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw' # UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw'
@@ -195,7 +197,7 @@ class DevConfig(Config):
# UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general' # UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general'
# SocketIO settings # SocketIO settings
SOCKETIO_MESSAGE_QUEUE = 'redis://redis:6379/1' SOCKETIO_MESSAGE_QUEUE = f'{REDIS_BASE_URI}/1'
SOCKETIO_CORS_ALLOWED_ORIGINS = '*' SOCKETIO_CORS_ALLOWED_ORIGINS = '*'
SOCKETIO_LOGGER = True SOCKETIO_LOGGER = True
SOCKETIO_ENGINEIO_LOGGER = True SOCKETIO_ENGINEIO_LOGGER = True
@@ -211,7 +213,7 @@ class DevConfig(Config):
GC_CRYPTO_KEY = 'envelope-encryption-key' GC_CRYPTO_KEY = 'envelope-encryption-key'
# Session settings # Session settings
SESSION_REDIS = redis.from_url('redis://redis:6379/2') SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')
# PATH settings # PATH settings
ffmpeg_path = '/usr/bin/ffmpeg' ffmpeg_path = '/usr/bin/ffmpeg'
@@ -278,6 +280,8 @@ class ProdConfig(Config):
# eveai_chat Redis Settings # eveai_chat Redis Settings
CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3' CELERY_BROKER_URL_CHAT = f'{REDIS_BASE_URI}/3'
CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3' CELERY_RESULT_BACKEND_CHAT = f'{REDIS_BASE_URI}/3'
# eveai_chat_workers cache Redis Settings
CHAT_WORKER_CACHE_URL = f'{REDIS_BASE_URI}/4'
# Session settings # Session settings
SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2') SESSION_REDIS = redis.from_url(f'{REDIS_BASE_URI}/2')

234
config/logging_config.py
View File

@@ -1,4 +1,8 @@
import json
import os import os
from datetime import datetime as dt, timezone as tz
from flask import current_app
from graypy import GELFUDPHandler from graypy import GELFUDPHandler
import logging import logging
import logging.config import logging.config
@@ -9,24 +13,173 @@ GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201))
env = os.environ.get('FLASK_ENV', 'development') env = os.environ.get('FLASK_ENV', 'development')
class CustomLogRecord(logging.LogRecord): class TuningLogRecord(logging.LogRecord):
"""Extended LogRecord that handles both tuning and business event logging"""
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
# Initialize extra fields after parent initialization
self._extra_fields = {}
self._is_tuning_log = False
self._tuning_type = None
self._tuning_tenant_id = None
self._tuning_catalog_id = None
self._tuning_specialist_id = None
self._tuning_retriever_id = None
self._tuning_processor_id = None
self.component = os.environ.get('COMPONENT_NAME', 'eveai_app') self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')
def __setattr__(self, name, value): def getMessage(self):
if name not in {'event_type', 'tenant_id', 'trace_id', 'span_id', 'span_name', 'parent_span_id', """
'document_version_id', 'chat_session_id', 'interaction_id', 'environment'}: Override getMessage to handle both string and dict messages
super().__setattr__(name, value) """
msg = self.msg
if self.args:
msg = msg % self.args
return msg
@property
def is_tuning_log(self):
return self._is_tuning_log
@is_tuning_log.setter
def is_tuning_log(self, value):
object.__setattr__(self, '_is_tuning_log', value)
@property
def tuning_type(self):
return self._tuning_type
@tuning_type.setter
def tuning_type(self, value):
object.__setattr__(self, '_tuning_type', value)
def get_tuning_data(self):
"""Get all tuning-related data if this is a tuning log"""
if not self._is_tuning_log:
return {}
return {
'is_tuning_log': self._is_tuning_log,
'tuning_type': self._tuning_type,
'tuning_tenant_id': self._tuning_tenant_id,
'tuning_catalog_id': self._tuning_catalog_id,
'tuning_specialist_id': self._tuning_specialist_id,
'tuning_retriever_id': self._tuning_retriever_id,
'tuning_processor_id': self._tuning_processor_id,
}
def set_tuning_data(self, tenant_id=None, catalog_id=None, specialist_id=None,
retriever_id=None, processor_id=None):
"""Set tuning-specific data"""
object.__setattr__(self, '_tuning_tenant_id', tenant_id)
object.__setattr__(self, '_tuning_catalog_id', catalog_id)
object.__setattr__(self, '_tuning_specialist_id', specialist_id)
object.__setattr__(self, '_tuning_retriever_id', retriever_id)
object.__setattr__(self, '_tuning_processor_id', processor_id)
def custom_log_record_factory(*args, **kwargs): class TuningFormatter(logging.Formatter):
record = CustomLogRecord(*args, **kwargs) """Universal formatter for all tuning logs"""
return record
def __init__(self, fmt=None, datefmt=None):
super().__init__(fmt or '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
datefmt or '%Y-%m-%d %H:%M:%S')
def format(self, record):
# First format with the default formatter to handle basic fields
formatted_msg = super().format(record)
# If this is a tuning log, add the additional context
if getattr(record, 'is_tuning_log', False):
try:
identifiers = []
if hasattr(record, 'tenant_id') and record.tenant_id:
identifiers.append(f"Tenant: {record.tenant_id}")
if hasattr(record, 'catalog_id') and record.catalog_id:
identifiers.append(f"Catalog: {record.catalog_id}")
if hasattr(record, 'processor_id') and record.processor_id:
identifiers.append(f"Processor: {record.processor_id}")
formatted_msg = (
f"{formatted_msg}\n"
f"[TUNING {record.tuning_type}] [{' | '.join(identifiers)}]"
)
if hasattr(record, 'tuning_data') and record.tuning_data:
formatted_msg += f"\nData: {json.dumps(record.tuning_data, indent=2)}"
except Exception as e:
return f"{formatted_msg} (Error formatting tuning data: {str(e)})"
return formatted_msg
class GraylogFormatter(logging.Formatter):
"""Maintains existing Graylog formatting while adding tuning fields"""
def format(self, record):
if getattr(record, 'is_tuning_log', False):
# Add tuning-specific fields to Graylog
record.tuning_fields = {
'is_tuning_log': True,
'tuning_type': record.tuning_type,
'tenant_id': record.tenant_id,
'catalog_id': record.catalog_id,
'specialist_id': record.specialist_id,
'retriever_id': record.retriever_id,
'processor_id': record.processor_id,
}
return super().format(record)
class TuningLogger:
"""Helper class to manage tuning logs with consistent structure"""
def __init__(self, logger_name, tenant_id=None, catalog_id=None, specialist_id=None, retriever_id=None, processor_id=None):
self.logger = logging.getLogger(logger_name)
self.tenant_id = tenant_id
self.catalog_id = catalog_id
self.specialist_id = specialist_id
self.retriever_id = retriever_id
self.processor_id = processor_id
def log_tuning(self, tuning_type: str, message: str, data=None, level=logging.DEBUG):
"""Log a tuning event with structured data"""
try:
# Create a standard LogRecord for tuning
record = logging.LogRecord(
name=self.logger.name,
level=level,
pathname='',
lineno=0,
msg=message,
args=(),
exc_info=None
)
# Add tuning-specific attributes
record.is_tuning_log = True
record.tuning_type = tuning_type
record.tenant_id = self.tenant_id
record.catalog_id = self.catalog_id
record.specialist_id = self.specialist_id
record.retriever_id = self.retriever_id
record.processor_id = self.processor_id
if data:
record.tuning_data = data
# Process the record
self.logger.handle(record)
except Exception as e:
fallback_logger = logging.getLogger('eveai_workers')
fallback_logger.exception(f"Failed to log tuning message: {str(e)}")
# Set the custom log record factory # Set the custom log record factory
logging.setLogRecordFactory(custom_log_record_factory) logging.setLogRecordFactory(TuningLogRecord)
LOGGING = { LOGGING = {
@@ -38,7 +191,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_app.log', 'filename': 'logs/eveai_app.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_workers': { 'file_workers': {
@@ -46,7 +199,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_workers.log', 'filename': 'logs/eveai_workers.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_chat': { 'file_chat': {
@@ -54,7 +207,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_chat.log', 'filename': 'logs/eveai_chat.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_chat_workers': { 'file_chat_workers': {
@@ -62,7 +215,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_chat_workers.log', 'filename': 'logs/eveai_chat_workers.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_api': { 'file_api': {
@@ -70,7 +223,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_api.log', 'filename': 'logs/eveai_api.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_beat': { 'file_beat': {
@@ -78,7 +231,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_beat.log', 'filename': 'logs/eveai_beat.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_entitlements': { 'file_entitlements': {
@@ -86,7 +239,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/eveai_entitlements.log', 'filename': 'logs/eveai_entitlements.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_sqlalchemy': { 'file_sqlalchemy': {
@@ -94,7 +247,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/sqlalchemy.log', 'filename': 'logs/sqlalchemy.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_mailman': { 'file_mailman': {
@@ -102,7 +255,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/mailman.log', 'filename': 'logs/mailman.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_security': { 'file_security': {
@@ -110,7 +263,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/security.log', 'filename': 'logs/security.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_rag_tuning': { 'file_rag_tuning': {
@@ -118,7 +271,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/rag_tuning.log', 'filename': 'logs/rag_tuning.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_embed_tuning': { 'file_embed_tuning': {
@@ -126,7 +279,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/embed_tuning.log', 'filename': 'logs/embed_tuning.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'file_business_events': { 'file_business_events': {
@@ -134,7 +287,7 @@ LOGGING = {
'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/business_events.log', 'filename': 'logs/business_events.log',
'maxBytes': 1024 * 1024 * 1, # 1MB 'maxBytes': 1024 * 1024 * 1, # 1MB
'backupCount': 10, 'backupCount': 2,
'formatter': 'standard', 'formatter': 'standard',
}, },
'console': { 'console': {
@@ -142,25 +295,38 @@ LOGGING = {
'level': 'DEBUG', 'level': 'DEBUG',
'formatter': 'standard', 'formatter': 'standard',
}, },
'tuning_file': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'logs/tuning.log',
'maxBytes': 1024 * 1024 * 3, # 3MB
'backupCount': 3,
'formatter': 'tuning',
},
'graylog': { 'graylog': {
'level': 'DEBUG', 'level': 'DEBUG',
'class': 'graypy.GELFUDPHandler', 'class': 'graypy.GELFUDPHandler',
'host': GRAYLOG_HOST, 'host': GRAYLOG_HOST,
'port': GRAYLOG_PORT, 'port': GRAYLOG_PORT,
'debugging_fields': True, # Set to True if you want to include debugging fields 'debugging_fields': True,
'extra_fields': True, # Set to True if you want to include extra fields 'formatter': 'graylog'
}, },
}, },
'formatters': { 'formatters': {
'standard': { 'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' 'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d]: %(message)s',
'[Thread: %(threadName)s]: %(message)s' 'datefmt': '%Y-%m-%d %H:%M:%S'
}, },
'graylog': { 'graylog': {
'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' 'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
'[Thread: %(threadName)s]: %(message)s', '[Thread: %(threadName)s]: %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S', 'datefmt': '%Y-%m-%d %H:%M:%S',
'()': GraylogFormatter
}, },
'tuning': {
'()': TuningFormatter,
'datefmt': '%Y-%m-%d %H:%M:%S UTC'
}
}, },
'loggers': { 'loggers': {
'eveai_app': { # logger for the eveai_app 'eveai_app': { # logger for the eveai_app
@@ -213,21 +379,17 @@ LOGGING = {
'level': 'DEBUG', 'level': 'DEBUG',
'propagate': False 'propagate': False
}, },
'rag_tuning': { # logger for the rag_tuning
'handlers': ['file_rag_tuning', 'graylog', ] if env == 'production' else ['file_rag_tuning', ],
'level': 'DEBUG',
'propagate': False
},
'embed_tuning': { # logger for the embed_tuning
'handlers': ['file_embed_tuning', 'graylog', ] if env == 'production' else ['file_embed_tuning', ],
'level': 'DEBUG',
'propagate': False
},
'business_events': { 'business_events': {
'handlers': ['file_business_events', 'graylog'], 'handlers': ['file_business_events', 'graylog'],
'level': 'DEBUG', 'level': 'DEBUG',
'propagate': False 'propagate': False
}, },
# Single tuning logger
'tuning': {
'handlers': ['tuning_file', 'graylog'] if env == 'production' else ['tuning_file'],
'level': 'DEBUG',
'propagate': False,
},
'': { # root logger '': { # root logger
'handlers': ['console'], 'handlers': ['console'],
'level': 'WARNING', # Set higher level for root to minimize noise 'level': 'WARNING', # Set higher level for root to minimize noise

56
config/processor_types.py Normal file
View File

@@ -0,0 +1,56 @@
# Catalog Types
PROCESSOR_TYPES = {
"HTML_PROCESSOR": {
"name": "HTML Processor",
"file_types": "html",
"Description": "A processor for HTML files",
"configuration": {
"html_tags": {
"name": "HTML Tags",
"type": "string",
"description": "A comma-separated list of HTML tags",
"required": True,
"default": "p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td"
},
"html_end_tags": {
"name": "HTML End Tags",
"type": "string",
"description": "A comma-separated list of HTML end tags (where can the chunk end)",
"required": True,
"default": "p, li, table"
},
"html_included_elements": {
"name": "HTML Included Elements",
"type": "string",
"description": "A comma-separated list of elements to be included",
"required": True,
"default": "article, main"
},
"html_excluded_elements": {
"name": "HTML Excluded Elements",
"type": "string",
"description": "A comma-separated list of elements to be excluded",
"required": False,
"default": "header, footer, nav, script"
},
"html_excluded_classes": {
"name": "HTML Excluded Classes",
"type": "string",
"description": "A comma-separated list of classes to be excluded",
"required": False,
},
},
},
"PDF_PROCESSOR": {
"name": "PDF Processor",
"file_types": "pdf",
"Description": "A Processor for PDF files",
"configuration": {}
},
"AUDIO_PROCESSOR": {
"name": "AUDIO Processor",
"file_types": "mp3, mp4, ogg",
"Description": "A Processor for audio files",
"configuration": {}
},
}

88
config/prompts/anthropic/claude-3-5-sonnet.yaml
View File

@@ -1,88 +0,0 @@
html_parse: |
You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
# Best practices are:
- Respect wordings and language(s) used in the HTML.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text.
HTML is between triple backticks.
```{html}```
pdf_parse: |
You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
# Best practices are:
- Respect wordings and language(s) used in the PDF.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- When headings are numbered, show the numbering and define the header level.
- A new item is started when a <return> is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily).
- Paragraphs are to be stripped of newlines so they become easily readable.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text.
PDF content is between triple backticks.
```{pdf_content}```
summary: |
Write a concise summary of the text in {language}. The text is delimited between triple backticks.
```{text}```
rag: |
Answer the question based on the following context, delimited between triple backticks.
{tenant_context}
Use the following {language} in your communication, and cite the sources used.
If the question cannot be answered using the given context, say "I have insufficient information to answer this question."
Context:
```{context}```
Question:
{question}
history: |
You are a helpful assistant that details a question based on a previous context,
in such a way that the question is understandable without the previous context.
The context is a conversation history, with the HUMAN asking questions, the AI answering questions.
The history is delimited between triple backticks.
You answer by stating the question in {language}.
History:
```{history}```
Question to be detailed:
{question}
encyclopedia: |
You have a lot of background knowledge, and as such you are some kind of
'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question.
If not, say you do not have sufficient information to answer the question. Use the {language} in your communication.
Question:
{question}
transcript: |
"""You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. Your task is to process and improve the given transcript, not to summarize it.
IMPORTANT INSTRUCTIONS:
1. DO NOT summarize the transcript and don't make your own interpretations. Return the FULL, COMPLETE transcript with improvements.
2. Improve any errors in the transcript based on context.
3. Respect the original wording and language(s) used in the transcription. Main Language used is {language}.
4. Divide the transcript into paragraphs for better readability. Each paragraph ONLY contains ORIGINAL TEXT.
5. Group related paragraphs into logical sections.
6. Add appropriate headers (using markdown syntax) to each section in {language}.
7. We do not need an overall title. Just add logical headers
8. Ensure that the entire transcript is included in your response, from start to finish.
REMEMBER:
- Your output should be the complete transcript in markdown format, NOT A SUMMARY OR ANALYSIS.
- Include EVERYTHING from the original transcript, just organized and formatted better.
- Just return the markdown version of the transcript, without any other text such as an introduction or a summary.
Here is the transcript to process (between triple backticks):
```{transcript}```
Process this transcript according to the instructions above and return the full, formatted markdown version.
"""

79
config/prompts/openai/gpt-4o-mini.yaml
View File

@@ -1,79 +0,0 @@
html_parse: |
You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
# Best practices are:
- Respect wordings and language(s) used in the HTML.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text.
HTML is between triple backquotes.
```{html}```
pdf_parse: |
You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
# Best practices are:
- Respect wordings and language(s) used in the PDF.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- When headings are numbered, show the numbering and define the header level.
- A new item is started when a <return> is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily).
- Paragraphs are to be stripped of newlines so they become easily readable.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text.
PDF content is between triple backquotes.
```{pdf_content}```
summary: |
Write a concise summary of the text in {language}. The text is delimited between triple backquotes.
```{text}```
rag: |
Answer the question based on the following context, delimited between triple backquotes.
{tenant_context}
Use the following {language} in your communication, and cite the sources used.
If the question cannot be answered using the given context, say "I have insufficient information to answer this question."
Context:
```{context}```
Question:
{question}
history: |
You are a helpful assistant that details a question based on a previous context,
in such a way that the question is understandable without the previous context.
The context is a conversation history, with the HUMAN asking questions, the AI answering questions.
The history is delimited between triple backquotes.
You answer by stating the question in {language}.
History:
```{history}```
Question to be detailed:
{question}
encyclopedia: |
You have a lot of background knowledge, and as such you are some kind of
'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question.
If not, say you do not have sufficient information to answer the question. Use the {language} in your communication.
Question:
{question}
transcript: |
You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
# Best practices and steps are:
- Respect wordings and language(s) used in the transcription. Main language is {language}.
- Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.
- Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part.
- annotate the text to identify these logical parts using headings in {language}.
- improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text.
The transcript is between triple backquotes.
```{transcript}```

84
config/prompts/openai/gpt-4o.yaml
View File

@@ -1,84 +0,0 @@
html_parse: |
You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
# Best practices are:
- Respect wordings and language(s) used in the HTML.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text.
HTML is between triple backquotes.
```{html}```
pdf_parse: |
You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
The content you get is already processed (some markdown already generated), but needs to be corrected. For large files, you may receive only portions of the full file. Consider this when processing the content.
# Best practices are:
- Respect wordings and language(s) used in the provided content.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- When headings are numbered, show the numbering and define the header level. You may have to correct current header levels, as preprocessing is known to make errors.
- A new item is started when a <return> is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily).
- Paragraphs are to be stripped of newlines so they become easily readable.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text.
PDF content is between triple backquotes.
```{pdf_content}```
summary: |
Write a concise summary of the text in {language}. The text is delimited between triple backquotes.
```{text}```
rag: |
Answer the question based on the following context, delimited between triple backquotes.
{tenant_context}
Use the following {language} in your communication, and cite the sources used.
If the question cannot be answered using the given context, say "I have insufficient information to answer this question."
Context:
```{context}```
Question:
{question}
history: |
You are a helpful assistant that details a question based on a previous context,
in such a way that the question is understandable without the previous context.
The context is a conversation history, with the HUMAN asking questions, the AI answering questions.
The history is delimited between triple backquotes.
You answer by stating the question in {language}.
History:
```{history}```
Question to be detailed:
{question}
encyclopedia: |
You have a lot of background knowledge, and as such you are some kind of
'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question.
If not, say you do not have sufficient information to answer the question. Use the {language} in your communication.
Question:
{question}
transcript: |
You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk.
# Best practices and steps are:
- Respect wordings and language(s) used in the transcription. Main language is {language}.
- Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.
- Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences.
- annotate the text to identify these logical parts using headings in {language}.
- improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text.
The transcript is between triple backquotes.
$$${previous_part}$$$
```{transcript}```

12
config/prompts/openai/gpt-4o/encyclopedia/1.0.0.yaml Normal file
View File

@@ -0,0 +1,12 @@
version: "1.0.0"
content: |
You have a lot of background knowledge, and as such you are some kind of
'encyclopedia' to explain general terminology. Only answer if you have a clear understanding of the question.
If not, say you do not have sufficient information to answer the question. Use the {language} in your communication.
Question:
{question}
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "A background information retriever for Evie"
changes: "Initial version migrated from flat file structure"

16
config/prompts/openai/gpt-4o/history/1.0.0.yaml Normal file
View File

@@ -0,0 +1,16 @@
version: "1.0.0"
content: |
You are a helpful assistant that details a question based on a previous context,
in such a way that the question is understandable without the previous context.
The context is a conversation history, with the HUMAN asking questions, the AI answering questions.
The history is delimited between triple backquotes.
You answer by stating the question in {language}.
History:
```{history}```
Question to be detailed:
{question}
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "Prompt to further detail a question based on the previous conversation"
changes: "Initial version migrated from flat file structure"

20
config/prompts/openai/gpt-4o/html_parse/1.0.0.yaml Normal file
View File

@@ -0,0 +1,20 @@
version: "1.0.0"
content: |
You are a top administrative assistant specialized in transforming given HTML into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
# Best practices are:
- Respect wordings and language(s) used in the HTML.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- Sub-headers can be used as lists. This is true when a header is followed by a series of sub-headers without content (paragraphs or listed items). Present those sub-headers as a list.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input html file. Answer with the pure markdown, without any other text.
HTML is between triple backquotes.
```{html}```
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "An aid in transforming HTML-based inputs to markdown"
changes: "Initial version migrated from flat file structure"

23
config/prompts/openai/gpt-4o/pdf_parse/1.0.0.yaml Normal file
View File

@@ -0,0 +1,23 @@
version: "1.0.0"
content: |
You are a top administrative aid specialized in transforming given PDF-files into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system.
The content you get is already processed (some markdown already generated), but needs to be corrected. For large files, you may receive only portions of the full file. Consider this when processing the content.
# Best practices are:
- Respect wordings and language(s) used in the provided content.
- The following items need to be considered: headings, paragraphs, listed items (numbered or not) and tables. Images can be neglected.
- When headings are numbered, show the numbering and define the header level. You may have to correct current header levels, as preprocessing is known to make errors.
- A new item is started when a <return> is found before a full line is reached. In order to know the number of characters in a line, please check the document and the context within the document (e.g. an image could limit the number of characters temporarily).
- Paragraphs are to be stripped of newlines so they become easily readable.
- Be careful of encoding of the text. Everything needs to be human readable.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of the processing of the complete input pdf content. Answer with the pure markdown, without any other text.
PDF content is between triple backquotes.
```{pdf_content}```
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "A assistant to parse PDF-content into markdown"
changes: "Initial version migrated from flat file structure"

15
config/prompts/openai/gpt-4o/rag/1.0.0.yaml Normal file
View File

@@ -0,0 +1,15 @@
version: "1.0.0"
content: |
Answer the question based on the following context, delimited between triple backquotes.
{tenant_context}
Use the following {language} in your communication, and cite the sources used at the end of the full conversation.
If the question cannot be answered using the given context, say "I have insufficient information to answer this question."
Context:
```{context}```
Question:
{question}
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "The Main RAG retriever"
changes: "Initial version migrated from flat file structure"

9
config/prompts/openai/gpt-4o/summary/1.0.0.yaml Normal file
View File

@@ -0,0 +1,9 @@
version: "1.0.0"
content: |
Write a concise summary of the text in {language}. The text is delimited between triple backquotes.
```{text}```
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "An assistant to create a summary when multiple chunks are required for 1 file"
changes: "Initial version migrated from flat file structure"

25
config/prompts/openai/gpt-4o/transcript/1.0.0.yaml Normal file
View File

@@ -0,0 +1,25 @@
version: "1.0.0"
content: |
You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk.
# Best practices and steps are:
- Respect wordings and language(s) used in the transcription. Main language is {language}.
- Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.
- Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences.
- annotate the text to identify these logical parts using headings in {language}.
- improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.
Process the file carefully, and take a stepped approach. The resulting markdown should be the result of processing the complete input transcription. Answer with the pure markdown, without any other text.
The transcript is between triple backquotes.
$$${previous_part}$$$
```{transcript}```
metadata:
author: "Josako"
date_added: "2024-11-10"
description: "An assistant to transform a transcript to markdown."
changes: "Initial version migrated from flat file structure"

8
config/retriever_types.py
View File

@@ -18,6 +18,14 @@ RETRIEVER_TYPES = {
"required": True, "required": True,
"default": 0.3, "default": 0.3,
}, },
},
"arguments": {
"query": {
"name": "query",
"type": "str",
"description": "Query to retrieve embeddings",
"required": True,
},
} }
} }
} }

41
config/specialist_types.py
View File

@@ -10,6 +10,13 @@ SPECIALIST_TYPES = {
"description": "The context to be used by the specialist.", "description": "The context to be used by the specialist.",
"required": False, "required": False,
}, },
"temperature": {
"name": "Temperature",
"type": "number",
"description": "The inference temperature to be used by the specialist.",
"required": False,
"default": 0.3
}
}, },
"arguments": { "arguments": {
"language": { "language": {
@@ -18,6 +25,38 @@ SPECIALIST_TYPES = {
"description": "Language code to be used for receiving questions and giving answers", "description": "Language code to be used for receiving questions and giving answers",
"required": True, "required": True,
}, },
"query": {
"name": "query",
"type": "str",
"description": "Query to answer",
"required": True,
}
},
"results": {
"detailed_query": {
"name": "detailed_query",
"type": "str",
"description": "The query detailed with the Chat Session History.",
"required": True,
},
"answer": {
"name": "answer",
"type": "str",
"description": "Answer to the query",
"required": True,
},
"citations": {
"name": "citations",
"type": "List[str]",
"description": "List of citations",
"required": False,
},
"insufficient_info": {
"name": "insufficient_info",
"type": "bool",
"description": "Whether or not the query is insufficient info",
"required": True,
},
} }
} }
} }

19
docker/compose_dev.yaml
View File

@@ -27,7 +27,6 @@ x-common-variables: &common-variables
OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7' OPENAI_API_KEY: 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7'
GROQ_API_KEY: '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' GROQ_API_KEY: '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'
ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2' ANTHROPIC_API_KEY: 'sk-ant-api03-c2TmkzbReeGhXBO5JxNH6BJNylRDonc9GmZd0eRbrvyekec2'
PORTKEY_API_KEY: 'T2Dt4QTpgCvWxa1OftYCJtj7NcDZ'
JWT_SECRET_KEY: 'bsdMkmQ8ObfMD52yAFg4trrvjgjMhuIqg2fjDpD/JqvgY0ccCcmlsEnVFmR79WPiLKEA3i8a5zmejwLZKl4v9Q==' JWT_SECRET_KEY: 'bsdMkmQ8ObfMD52yAFg4trrvjgjMhuIqg2fjDpD/JqvgY0ccCcmlsEnVFmR79WPiLKEA3i8a5zmejwLZKl4v9Q=='
API_ENCRYPTION_KEY: 'xfF5369IsredSrlrYZqkM9ZNrfUASYYS6TCcAR9UKj4=' API_ENCRYPTION_KEY: 'xfF5369IsredSrlrYZqkM9ZNrfUASYYS6TCcAR9UKj4='
MINIO_ENDPOINT: minio:9000 MINIO_ENDPOINT: minio:9000
@@ -90,7 +89,7 @@ services:
- ../migrations:/app/migrations - ../migrations:/app/migrations
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -124,7 +123,7 @@ services:
- ../config:/app/config - ../config:/app/config
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -154,7 +153,7 @@ services:
- ../config:/app/config - ../config:/app/config
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -186,7 +185,7 @@ services:
- ../config:/app/config - ../config:/app/config
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -214,7 +213,7 @@ services:
- ../config:/app/config - ../config:/app/config
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -248,7 +247,7 @@ services:
- ../config:/app/config - ../config:/app/config
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
redis: redis:
condition: service_healthy condition: service_healthy
@@ -272,7 +271,7 @@ services:
- ../config:/app/config - ../config:/app/config
- ../scripts:/app/scripts - ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages - ../patched_packages:/app/patched_packages
- eveai_logs:/app/logs - ./eveai_logs:/app/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -308,8 +307,8 @@ services:
redis: redis:
image: redis:7.2.5 image: redis:7.2.5
restart: always restart: always
expose: ports:
- 6379 - "6379:6379"
volumes: volumes:
- ./db/redis:/data - ./db/redis:/data
healthcheck: healthcheck:

1
docker/compose_stackhero.yaml
View File

@@ -31,7 +31,6 @@ x-common-variables: &common-variables
OPENAI_API_KEY: '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' OPENAI_API_KEY: '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'
GROQ_API_KEY: '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' GROQ_API_KEY: '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'
ANTHROPIC_API_KEY: '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' ANTHROPIC_API_KEY: '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'
PORTKEY_API_KEY: 'XvmvBFIVbm76opUxA7MNP14QmdQj'
JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408' JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408'
API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0=' API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0='
GRAYLOG_HOST: de4zvu.stackhero-network.com GRAYLOG_HOST: de4zvu.stackhero-network.com

23
eveai_api/__init__.py
View File

@@ -52,37 +52,16 @@ def create_app(config_file=None):
@app.before_request @app.before_request
def before_request(): def before_request():
app.logger.debug(f'Before request: {request.method} {request.path}')
app.logger.debug(f'Request URL: {request.url}')
app.logger.debug(f'Request headers: {dict(request.headers)}')
# Log JSON data if the content type is application/json
if request.is_json:
app.logger.debug(f'JSON data: {request.json}')
# Log raw data for other content types
if request.data:
app.logger.debug(f'Raw data: {request.data}')
# Check if this is a request to the token endpoint
if request.path == '/api/v1/auth/token' and request.method == 'POST':
app.logger.debug('Token request detected, skipping JWT verification')
return
# Check if this a health check request # Check if this a health check request
if request.path.startswith('/_healthz') or request.path.startswith('/healthz'): if request.path.startswith('/_healthz') or request.path.startswith('/healthz'):
app.logger.debug('Health check request detected, skipping JWT verification') pass
else: else:
try: try:
verify_jwt_in_request(optional=True) verify_jwt_in_request(optional=True)
tenant_id = get_jwt_identity() tenant_id = get_jwt_identity()
app.logger.debug(f'Tenant ID from JWT: {tenant_id}')
if tenant_id: if tenant_id:
Database(tenant_id).switch_schema() Database(tenant_id).switch_schema()
app.logger.debug(f'Switched to schema for tenant {tenant_id}')
else:
app.logger.debug('No tenant ID found in JWT')
except Exception as e: except Exception as e:
app.logger.error(f'Error in before_request: {str(e)}') app.logger.error(f'Error in before_request: {str(e)}')
# Don't raise the exception here, let the request continue # Don't raise the exception here, let the request continue

8
eveai_api/api/auth.py
View File

@@ -30,8 +30,6 @@ class Token(Resource):
""" """
Get JWT token Get JWT token
""" """
current_app.logger.debug(f"Token endpoint called with data: {request.json}")
try: try:
tenant_id = auth_ns.payload['tenant_id'] tenant_id = auth_ns.payload['tenant_id']
api_key = auth_ns.payload['api_key'] api_key = auth_ns.payload['api_key']
@@ -39,17 +37,13 @@ class Token(Resource):
current_app.logger.error(f"Missing required field: {e}") current_app.logger.error(f"Missing required field: {e}")
return {'message': f"Missing required field: {e}"}, 400 return {'message': f"Missing required field: {e}"}, 400
current_app.logger.debug(f"Querying database for tenant: {tenant_id}")
tenant = Tenant.query.get(tenant_id) tenant = Tenant.query.get(tenant_id)
if not tenant: if not tenant:
current_app.logger.error(f"Tenant not found: {tenant_id}") current_app.logger.error(f"Tenant not found: {tenant_id}")
return {'message': "Tenant not found"}, 404 return {'message': "Tenant not found"}, 404
current_app.logger.debug(f"Tenant found: {tenant.id}")
try: try:
current_app.logger.debug("Attempting to decrypt API key")
decrypted_api_key = simple_encryption.decrypt_api_key(tenant.encrypted_api_key) decrypted_api_key = simple_encryption.decrypt_api_key(tenant.encrypted_api_key)
except Exception as e: except Exception as e:
current_app.logger.error(f"Error decrypting API key: {e}") current_app.logger.error(f"Error decrypting API key: {e}")
@@ -63,9 +57,7 @@ class Token(Resource):
expires_delta = current_app.config.get('JWT_ACCESS_TOKEN_EXPIRES', timedelta(minutes=15)) expires_delta = current_app.config.get('JWT_ACCESS_TOKEN_EXPIRES', timedelta(minutes=15))
try: try:
current_app.logger.debug(f"Creating access token for tenant: {tenant_id}")
access_token = create_access_token(identity=tenant_id, expires_delta=expires_delta) access_token = create_access_token(identity=tenant_id, expires_delta=expires_delta)
current_app.logger.debug("Access token created successfully")
return { return {
'access_token': access_token, 'access_token': access_token,
'expires_in': expires_delta.total_seconds() 'expires_in': expires_delta.total_seconds()

3
eveai_app/__init__.py
View File

@@ -7,7 +7,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix
import logging.config import logging.config
from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session, from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session,
minio_client, simple_encryption, metrics) minio_client, simple_encryption, metrics, cache_manager)
from common.models.user import User, Role, Tenant, TenantDomain from common.models.user import User, Role, Tenant, TenantDomain
import common.models.interaction import common.models.interaction
import common.models.entitlements import common.models.entitlements
@@ -119,6 +119,7 @@ def register_extensions(app):
simple_encryption.init_app(app) simple_encryption.init_app(app)
session.init_app(app) session.init_app(app)
minio_client.init_app(app) minio_client.init_app(app)
cache_manager.init_app(app)
metrics.init_app(app) metrics.init_app(app)

33
eveai_app/templates/document/edit_processor.html Normal file
View File

@@ -0,0 +1,33 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% block title %}Edit Processor{% endblock %}
{% block content_title %}Edit Processor{% endblock %}
{% block content_description %}Edit a Processor (for a Catalog){% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = ['type'] %}
{% set exclude_fields = [] %}
<!-- Render Static Fields -->
{% for field in form.get_static_fields() %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<!-- Render Dynamic Fields -->
{% for collection_name, fields in form.get_dynamic_fields().items() %}
{% if fields|length > 0 %}
<h4 class="mt-4">{{ collection_name }}</h4>
{% endif %}
{% for field in fields %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
{% endfor %}
<button type="submit" class="btn btn-primary">Save Processor</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

23
eveai_app/templates/document/processor.html Normal file
View File

@@ -0,0 +1,23 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% block title %}Processor Registration{% endblock %}
{% block content_title %}Register Processor{% endblock %}
{% block content_description %}Define a new processor (for a catalog){% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Register Processor</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

23
eveai_app/templates/document/processors.html Normal file
View File

@@ -0,0 +1,23 @@
{% extends 'base.html' %}
{% from 'macros.html' import render_selectable_table, render_pagination %}
{% block title %}Processors{% endblock %}
{% block content_title %}Processors{% endblock %}
{% block content_description %}View Processors for Tenant{% endblock %}
{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
{% block content %}
<div class="container">
<form method="POST" action="{{ url_for('document_bp.handle_processor_selection') }}">
{{ render_selectable_table(headers=["Processor ID", "Name", "Type", "Catalog ID"], rows=rows, selectable=True, id="retrieversTable") }}
<div class="form-group mt-3">
<button type="submit" name="action" value="edit_processor" class="btn btn-primary">Edit Processor</button>
</div>
</form>
</div>
{% endblock %}
{% block content_footer %}
{{ render_pagination(pagination, 'document_bp.processors') }}
{% endblock %}

2
eveai_app/templates/interaction/edit_specialist.html
View File

@@ -24,7 +24,7 @@
{{ render_field(field, disabled_fields, exclude_fields) }} {{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
<button type="submit" class="btn btn-primary">Save Retriever</button> <button type="submit" class="btn btn-primary">Save Specialist</button>
</form> </form>
{% endblock %} {% endblock %}

2
eveai_app/templates/navbar.html
View File

@@ -83,6 +83,8 @@
{{ dropdown('Document Mgmt', 'note_stack', [ {{ dropdown('Document Mgmt', 'note_stack', [
{'name': 'Add Catalog', 'url': '/document/catalog', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'Add Catalog', 'url': '/document/catalog', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'All Catalogs', 'url': '/document/catalogs', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'All Catalogs', 'url': '/document/catalogs', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Add Processor', 'url': '/document/processor', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'All Processors', 'url': '/document/processors', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Add Retriever', 'url': '/document/retriever', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'Add Retriever', 'url': '/document/retriever', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'All Retrievers', 'url': '/document/retrievers', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'All Retrievers', 'url': '/document/retrievers', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Tenant Admin']}, {'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Tenant Admin']},

3
eveai_app/views/basic_views.py
View File

@@ -9,12 +9,11 @@ basic_bp = Blueprint('basic_bp', __name__)
@basic_bp.before_request @basic_bp.before_request
def log_before_request(): def log_before_request():
current_app.logger.debug(f"Before request (basic_bp): {request.method} {request.url}") pass
@basic_bp.after_request @basic_bp.after_request
def log_after_request(response): def log_after_request(response):
current_app.logger.debug(f"After request (basic_bp): {request.method} {request.url} - Status: {response.status}")
return response return response

117
eveai_app/views/document_forms.py
View File

@@ -11,6 +11,7 @@ from wtforms_sqlalchemy.fields import QuerySelectField
from common.models.document import Catalog from common.models.document import Catalog
from config.catalog_types import CATALOG_TYPES from config.catalog_types import CATALOG_TYPES
from config.processor_types import PROCESSOR_TYPES
from config.retriever_types import RETRIEVER_TYPES from config.retriever_types import RETRIEVER_TYPES
from .dynamic_form_base import DynamicFormBase from .dynamic_form_base import DynamicFormBase
@@ -38,28 +39,14 @@ class CatalogForm(FlaskForm):
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config) # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = SelectField('Catalog Type', validators=[DataRequired()]) type = SelectField('Catalog Type', validators=[DataRequired()])
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
# HTML Embedding Variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
default='p, li')
html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
default='header, footer, nav, script')
html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()], min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000) default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()], max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000) default=3000)
# Chat Variables
chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)]) # Metadata fields
chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
# Tuning variables system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@@ -74,28 +61,75 @@ class EditCatalogForm(DynamicFormBase):
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config) # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = StringField('Catalog Type', validators=[DataRequired()], render_kw={'readonly': True}) type = StringField('Catalog Type', validators=[DataRequired()], render_kw={'readonly': True})
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],)
# HTML Embedding Variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
default='p, li')
html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
default='header, footer, nav, script')
html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()], min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000) default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()], max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000) default=3000)
# Chat Variables
chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)]) # Metadata fields
chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
# Tuning variables system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],)
embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
class ProcessorForm(FlaskForm):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])
# Catalog for the Retriever
catalog = QuerySelectField(
'Catalog ID',
query_factory=lambda: Catalog.query.all(),
allow_blank=True,
get_label='name',
validators=[Optional()],
)
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = SelectField('Processor Type', validators=[DataRequired()])
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
tuning = BooleanField('Enable Embedding Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Dynamically populate the 'type' field using the constructor
self.type.choices = [(key, value['name']) for key, value in PROCESSOR_TYPES.items()]
class EditProcessorForm(DynamicFormBase):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])
# Catalog for the Retriever
catalog = QuerySelectField(
'Catalog ID',
query_factory=lambda: Catalog.query.all(),
allow_blank=True,
get_label='name',
validators=[Optional()],
)
type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True})
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
tuning = BooleanField('Enable Embedding Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
class RetrieverForm(FlaskForm): class RetrieverForm(FlaskForm):
@@ -135,22 +169,17 @@ class EditRetrieverForm(DynamicFormBase):
validators=[Optional()], validators=[Optional()],
) )
# Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config) # Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config)
type = SelectField('Retriever Type', validators=[DataRequired()], render_kw={'readonly': True}) type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True})
tuning = BooleanField('Enable Tuning', default=False) tuning = BooleanField('Enable Tuning', default=False)
# Metadata fields # Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Set the retriever type choices (loaded from config)
self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
class AddDocumentForm(DynamicFormBase): class AddDocumentForm(DynamicFormBase):
file = FileField('File', validators=[FileRequired(), allowed_file]) file = FileField('File', validators=[FileRequired(), allowed_file])
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
name = StringField('Name', validators=[Length(max=100)]) name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()]) language = SelectField('Language', choices=[], validators=[Optional()])
user_context = TextAreaField('User Context', validators=[Optional()]) user_context = TextAreaField('User Context', validators=[Optional()])
@@ -167,6 +196,7 @@ class AddDocumentForm(DynamicFormBase):
class AddURLForm(DynamicFormBase): class AddURLForm(DynamicFormBase):
url = URLField('URL', validators=[DataRequired(), URL()]) url = URLField('URL', validators=[DataRequired(), URL()])
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
name = StringField('Name', validators=[Length(max=100)]) name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()]) language = SelectField('Language', choices=[], validators=[Optional()])
user_context = TextAreaField('User Context', validators=[Optional()]) user_context = TextAreaField('User Context', validators=[Optional()])
@@ -207,6 +237,7 @@ class EditDocumentForm(FlaskForm):
class EditDocumentVersionForm(DynamicFormBase): class EditDocumentVersionForm(DynamicFormBase):
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
language = StringField('Language') language = StringField('Language')
user_context = TextAreaField('User Context', validators=[Optional()]) user_context = TextAreaField('User Context', validators=[Optional()])
system_context = TextAreaField('System Context', validators=[Optional()]) system_context = TextAreaField('System Context', validators=[Optional()])

172
eveai_app/views/document_views.py
View File

@@ -14,15 +14,16 @@ from urllib.parse import urlparse, unquote
import io import io
import json import json
from common.models.document import Document, DocumentVersion, Catalog, Retriever from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor
from common.extensions import db, minio_client from common.extensions import db, minio_client
from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \ from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
process_multiple_urls, get_documents_list, edit_document, \ process_multiple_urls, get_documents_list, edit_document, \
edit_document_version, refresh_document edit_document_version, refresh_document
from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \ from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
EveAIDoubleURLException EveAIDoubleURLException
from config.processor_types import PROCESSOR_TYPES
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \ from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
from common.utils.middleware import mw_before_request from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for from common.utils.nginx_utils import prefixed_url_for
@@ -37,13 +38,11 @@ document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@document_bp.before_request @document_bp.before_request
def log_before_request(): def log_before_request():
current_app.logger.debug(f"Before request (document_bp): {request.method} {request.url}") pass
@document_bp.after_request @document_bp.after_request
def log_after_request(response): def log_after_request(response):
current_app.logger.debug(
f"After request (document_bp): {request.method} {request.url} - Status: {response.status}")
return response return response
@@ -53,8 +52,6 @@ def before_request():
mw_before_request() mw_before_request()
except Exception as e: except Exception as e:
current_app.logger.error(f'Error switching schema in Document Blueprint: {e}') current_app.logger.error(f'Error switching schema in Document Blueprint: {e}')
for role in current_user.roles:
current_app.logger.debug(f'User {current_user.email} has role {role.name}')
raise raise
@@ -67,16 +64,6 @@ def catalog():
tenant_id = session.get('tenant').get('id') tenant_id = session.get('tenant').get('id')
new_catalog = Catalog() new_catalog = Catalog()
form.populate_obj(new_catalog) form.populate_obj(new_catalog)
# Handle Embedding Variables
new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
if form.html_end_tags.data else []
new_catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
if form.html_included_elements.data else []
new_catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
if form.html_excluded_elements.data else []
new_catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
set_logging_information(new_catalog, dt.now(tz.utc)) set_logging_information(new_catalog, dt.now(tz.utc))
try: try:
@@ -84,6 +71,8 @@ def catalog():
db.session.commit() db.session.commit()
flash('Catalog successfully added!', 'success') flash('Catalog successfully added!', 'success')
current_app.logger.info(f'Catalog {new_catalog.name} successfully added for tenant {tenant_id}!') current_app.logger.info(f'Catalog {new_catalog.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of catalog - add configuration of the catalog (dependent on type)
return redirect(prefixed_url_for('document_bp.catalog', catalog_id=new_catalog.id))
except SQLAlchemyError as e: except SQLAlchemyError as e:
db.session.rollback() db.session.rollback()
flash(f'Failed to add catalog. Error: {e}', 'danger') flash(f'Failed to add catalog. Error: {e}', 'danger')
@@ -140,27 +129,8 @@ def edit_catalog(catalog_id):
configuration_config = CATALOG_TYPES[catalog.type]["configuration"] configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, catalog.configuration) form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
# Convert arrays to comma-separated strings for display
if request.method == 'GET':
form.html_tags.data = ', '.join(catalog.html_tags or '')
form.html_end_tags.data = ', '.join(catalog.html_end_tags or '')
form.html_included_elements.data = ', '.join(catalog.html_included_elements or '')
form.html_excluded_elements.data = ', '.join(catalog.html_excluded_elements or '')
form.html_excluded_classes.data = ', '.join(catalog.html_excluded_classes or '')
if request.method == 'POST' and form.validate_on_submit(): if request.method == 'POST' and form.validate_on_submit():
form.populate_obj(catalog) form.populate_obj(catalog)
# Handle Embedding Variables
catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
if form.html_end_tags.data else []
catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
if form.html_included_elements.data else []
catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
if form.html_excluded_elements.data else []
catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
catalog.configuration = form.get_dynamic_data('configuration') catalog.configuration = form.get_dynamic_data('configuration')
update_logging_information(catalog, dt.now(tz.utc)) update_logging_information(catalog, dt.now(tz.utc))
try: try:
@@ -180,6 +150,116 @@ def edit_catalog(catalog_id):
return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id) return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id)
@document_bp.route('/processor', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def processor():
form = ProcessorForm()
if form.validate_on_submit():
tenant_id = session.get('tenant').get('id')
new_processor = Processor()
form.populate_obj(new_processor)
new_processor.catalog_id = form.catalog.data.id
set_logging_information(new_processor, dt.now(tz.utc))
try:
db.session.add(new_processor)
db.session.commit()
flash('Processor successfully added!', 'success')
current_app.logger.info(f'Processor {new_processor.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=new_processor.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add processor. Error: {e}', 'danger')
current_app.logger.error(f'Failed to add retriever {new_processor.name}'
f'for tenant {tenant_id}. Error: {str(e)}')
return render_template('document/processor.html', form=form)
@document_bp.route('/processor/<int:processor_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def edit_processor(processor_id):
"""Edit an existing processorr configuration."""
# Get the processor or return 404
processor = Processor.query.get_or_404(processor_id)
if processor.catalog_id:
# If catalog_id is just an ID, fetch the Catalog object
processor.catalog = Catalog.query.get(processor.catalog_id)
else:
processor.catalog = None
# Create form instance with the processor
form = EditProcessorForm(request.form, obj=processor)
configuration_config = PROCESSOR_TYPES[processor.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, processor.configuration)
if form.validate_on_submit():
# Update basic fields
form.populate_obj(processor)
processor.configuration = form.get_dynamic_data('configuration')
# Update catalog relationship
processor.catalog_id = form.catalog.data.id if form.catalog.data else None
# Update logging information
update_logging_information(processor, dt.now(tz.utc))
# Save changes to database
try:
db.session.add(processor)
db.session.commit()
flash('Retriever updated successfully!', 'success')
current_app.logger.info(f'Retriever {processor.id} updated successfully')
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to update processor. Error: {str(e)}', 'danger')
current_app.logger.error(f'Failed to update processor {processor_id}. Error: {str(e)}')
return render_template('document/edit_processor.html', form=form, processor_id=processor_id)
return redirect(prefixed_url_for('document_bp.processors'))
else:
form_validation_failed(request, form)
return render_template('document/edit_processor.html', form=form, processor_id=processor_id)
@document_bp.route('/processors', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def processors():
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = Processor.query.order_by(Processor.id)
pagination = query.paginate(page=page, per_page=per_page)
the_processors = pagination.items
# prepare table data
rows = prepare_table_for_macro(the_processors,
[('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')])
# Render the catalogs in a template
return render_template('document/processors.html', rows=rows, pagination=pagination)
@document_bp.route('/handle_processor_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def handle_processor_selection():
processor_identification = request.form.get('selected_row')
processor_id = ast.literal_eval(processor_identification).get('value')
action = request.form['action']
if action == 'edit_processor':
return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=processor_id))
return redirect(prefixed_url_for('document_bp.processors'))
@document_bp.route('/retriever', methods=['GET', 'POST']) @document_bp.route('/retriever', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin') @roles_accepted('Super User', 'Tenant Admin')
def retriever(): def retriever():
@@ -198,15 +278,14 @@ def retriever():
db.session.commit() db.session.commit()
flash('Retriever successfully added!', 'success') flash('Retriever successfully added!', 'success')
current_app.logger.info(f'Catalog {new_retriever.name} successfully added for tenant {tenant_id}!') current_app.logger.info(f'Catalog {new_retriever.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.edit_retriever', retriever_id=new_retriever.id))
except SQLAlchemyError as e: except SQLAlchemyError as e:
db.session.rollback() db.session.rollback()
flash(f'Failed to add retriever. Error: {e}', 'danger') flash(f'Failed to add retriever. Error: {e}', 'danger')
current_app.logger.error(f'Failed to add retriever {new_retriever.name}' current_app.logger.error(f'Failed to add retriever {new_retriever.name}'
f'for tenant {tenant_id}. Error: {str(e)}') f'for tenant {tenant_id}. Error: {str(e)}')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.retriever', retriever_id=new_retriever.id))
return render_template('document/retriever.html', form=form) return render_template('document/retriever.html', form=form)
@@ -311,6 +390,7 @@ def add_document():
current_app.logger.info(f'Adding Document for {catalog_id}') current_app.logger.info(f'Adding Document for {catalog_id}')
tenant_id = session['tenant']['id'] tenant_id = session['tenant']['id']
file = form.file.data file = form.file.data
sub_file_type = form.sub_file_type.data
filename = secure_filename(file.filename) filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower() extension = filename.rsplit('.', 1)[1].lower()
@@ -324,14 +404,13 @@ def add_document():
api_input = { api_input = {
'catalog_id': catalog_id, 'catalog_id': catalog_id,
'name': form.name.data, 'name': form.name.data,
'sub_file_type': form.sub_file_type.data,
'language': form.language.data, 'language': form.language.data,
'user_context': form.user_context.data, 'user_context': form.user_context.data,
'valid_from': form.valid_from.data, 'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None, 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
'catalog_properties': catalog_properties, 'catalog_properties': catalog_properties,
} }
current_app.logger.debug(f'Creating document stack with input {api_input}')
new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id) new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
task_id = start_embedding_task(tenant_id, new_doc_vers.id) task_id = start_embedding_task(tenant_id, new_doc_vers.id)
@@ -341,6 +420,7 @@ def add_document():
except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e: except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e:
flash(str(e), 'error') flash(str(e), 'error')
current_app.logger.error(f"Error adding document: {str(e)}")
except Exception as e: except Exception as e:
current_app.logger.error(f'Error adding document: {str(e)}') current_app.logger.error(f'Error adding document: {str(e)}')
flash('An error occurred while adding the document.', 'error') flash('An error occurred while adding the document.', 'error')
@@ -378,6 +458,7 @@ def add_url():
api_input = { api_input = {
'catalog_id': catalog_id, 'catalog_id': catalog_id,
'name': form.name.data or filename, 'name': form.name.data or filename,
'sub_file_type': form.sub_file_type.data,
'url': url, 'url': url,
'language': form.language.data, 'language': form.language.data,
'user_context': form.user_context.data, 'user_context': form.user_context.data,
@@ -562,8 +643,6 @@ def handle_document_version_selection():
action = request.form['action'] action = request.form['action']
current_app.logger.debug(f'Triggered Document Version Action: {action}')
match action: match action:
case 'edit_document_version': case 'edit_document_version':
return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=doc_vers_id)) return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=doc_vers_id))
@@ -598,9 +677,7 @@ def handle_library_selection():
@document_bp.route('/document_versions_list', methods=['GET']) @document_bp.route('/document_versions_list', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin') @roles_accepted('Super User', 'Tenant Admin')
def document_versions_list(): def document_versions_list():
current_app.logger.debug('Getting document versions list')
view = DocumentVersionListView(DocumentVersion, 'document/document_versions_list_view.html', per_page=20) view = DocumentVersionListView(DocumentVersion, 'document/document_versions_list_view.html', per_page=20)
current_app.logger.debug('Got document versions list')
return view.get() return view.get()
@@ -653,8 +730,9 @@ def update_logging_information(obj, timestamp):
def log_session_state(session, msg=""): def log_session_state(session, msg=""):
current_app.logger.debug(f"{msg} - Session dirty: {session.dirty}") pass
current_app.logger.debug(f"{msg} - Session new: {session.new}") # current_app.logger.info(f"{msg} - Session dirty: {session.dirty}")
# current_app.logger.info(f"{msg} - Session new: {session.new}")
def fetch_html(url): def fetch_html(url):

29
eveai_app/views/dynamic_form_base.py
View File

@@ -5,6 +5,7 @@ import json
from wtforms.fields.choices import SelectField from wtforms.fields.choices import SelectField
from wtforms.fields.datetime import DateField from wtforms.fields.datetime import DateField
from common.utils.config_field_types import TaggingFields
class DynamicFormBase(FlaskForm): class DynamicFormBase(FlaskForm):
@@ -38,14 +39,35 @@ class DynamicFormBase(FlaskForm):
message=f"Value must be between {min_value or '-∞'} and {max_value or '∞'}" message=f"Value must be between {min_value or '-∞'} and {max_value or '∞'}"
) )
) )
elif field_type == 'tagging_fields':
validators_list.append(self._validate_tagging_fields)
return validators_list return validators_list
def _validate_tagging_fields(self, form, field):
"""Validate the tagging fields structure"""
if not field.data:
return
try:
# Parse JSON data
fields_data = json.loads(field.data)
# Validate using TaggingFields model
try:
TaggingFields.from_dict(fields_data)
except ValueError as e:
raise ValidationError(str(e))
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except Exception as e:
raise ValidationError(f"Invalid field definition: {str(e)}")
def add_dynamic_fields(self, collection_name, config, initial_data=None): def add_dynamic_fields(self, collection_name, config, initial_data=None):
"""Add dynamic fields to the form based on the configuration.""" """Add dynamic fields to the form based on the configuration."""
self.dynamic_fields[collection_name] = [] self.dynamic_fields[collection_name] = []
for field_name, field_def in config.items(): for field_name, field_def in config.items():
current_app.logger.debug(f"{field_name}: {field_def}")
# Prefix the field name with the collection name # Prefix the field name with the collection name
full_field_name = f"{collection_name}_{field_name}" full_field_name = f"{collection_name}_{field_name}"
label = field_def.get('name', field_name) label = field_def.get('name', field_name)
@@ -59,7 +81,6 @@ class DynamicFormBase(FlaskForm):
# Handle special case for tagging_fields # Handle special case for tagging_fields
if field_type == 'tagging_fields': if field_type == 'tagging_fields':
field_class = TextAreaField field_class = TextAreaField
field_validators.append(validate_tagging_fields)
extra_classes = 'json-editor' extra_classes = 'json-editor'
field_kwargs = {} field_kwargs = {}
elif field_type == 'enum': elif field_type == 'enum':
@@ -145,16 +166,12 @@ class DynamicFormBase(FlaskForm):
def get_dynamic_data(self, collection_name): def get_dynamic_data(self, collection_name):
"""Retrieve the data from dynamic fields of a specific collection.""" """Retrieve the data from dynamic fields of a specific collection."""
data = {} data = {}
current_app.logger.debug(f"{collection_name} in {self.dynamic_fields}?")
if collection_name not in self.dynamic_fields: if collection_name not in self.dynamic_fields:
return data return data
prefix_length = len(collection_name) + 1 # +1 for the underscore prefix_length = len(collection_name) + 1 # +1 for the underscore
for full_field_name in self.dynamic_fields[collection_name]: for full_field_name in self.dynamic_fields[collection_name]:
current_app.logger.debug(f"{full_field_name}: {full_field_name}")
original_field_name = full_field_name[prefix_length:] original_field_name = full_field_name[prefix_length:]
current_app.logger.debug(f"{original_field_name}: {original_field_name}")
field = getattr(self, full_field_name) field = getattr(self, full_field_name)
current_app.logger.debug(f"{field}: {field}")
# Parse JSON for tagging_fields type # Parse JSON for tagging_fields type
if isinstance(field, TextAreaField) and field.data: if isinstance(field, TextAreaField) and field.data:
try: try:

1
eveai_app/views/entitlements_views.py
View File

@@ -159,7 +159,6 @@ def create_license(license_tier_id):
tenant_id=tenant_id, tenant_id=tenant_id,
tier_id=license_tier_id, tier_id=license_tier_id,
) )
current_app.logger.debug(f"Currency data in form: {form.currency.data}")
if form.validate_on_submit(): if form.validate_on_submit():
# Update the license with form data # Update the license with form data
form.populate_obj(new_license) form.populate_obj(new_license)

2
eveai_app/views/interaction_forms.py
View File

@@ -33,7 +33,7 @@ class SpecialistForm(FlaskForm):
type = SelectField('Specialist Type', validators=[DataRequired()]) type = SelectField('Specialist Type', validators=[DataRequired()])
tuning = BooleanField('Enable Retrieval Tuning', default=False) tuning = BooleanField('Enable Specialist Tuning', default=False)
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)

40
eveai_app/views/interaction_views.py
View File

@@ -32,13 +32,11 @@ interaction_bp = Blueprint('interaction_bp', __name__, url_prefix='/interaction'
@interaction_bp.before_request @interaction_bp.before_request
def log_before_request(): def log_before_request():
current_app.logger.debug(f"Before request (interaction_bp): {request.method} {request.url}") pass
@interaction_bp.after_request @interaction_bp.after_request
def log_after_request(response): def log_after_request(response):
current_app.logger.debug(
f"After request (interaction_bp): {request.method} {request.url} - Status: {response.status}")
return response return response
@@ -48,8 +46,6 @@ def before_request():
mw_before_request() mw_before_request()
except Exception as e: except Exception as e:
current_app.logger.error(f'Error switching schema in Interaction Blueprint: {e}') current_app.logger.error(f'Error switching schema in Interaction Blueprint: {e}')
for role in current_user.roles:
current_app.logger.debug(f'User {current_user.email} has role {role.name}')
raise raise
@@ -147,14 +143,9 @@ def specialist():
db.session.add(new_specialist) db.session.add(new_specialist)
db.session.flush() # This assigns the ID to the specialist without committing the transaction db.session.flush() # This assigns the ID to the specialist without committing the transaction
current_app.logger.debug(
f'New Specialist after flush - id: {new_specialist.id}, name: {new_specialist.name}')
# Create the retriever associations # Create the retriever associations
selected_retrievers = form.retrievers.data selected_retrievers = form.retrievers.data
current_app.logger.debug(f'Selected Retrievers - {selected_retrievers}')
for retriever in selected_retrievers: for retriever in selected_retrievers:
current_app.logger.debug(f'Creating association for Retriever - {retriever.id}')
specialist_retriever = SpecialistRetriever( specialist_retriever = SpecialistRetriever(
specialist_id=new_specialist.id, specialist_id=new_specialist.id,
retriever_id=retriever.id retriever_id=retriever.id
@@ -174,7 +165,7 @@ def specialist():
flash(f'Failed to add specialist. Error: {str(e)}', 'danger') flash(f'Failed to add specialist. Error: {str(e)}', 'danger')
return render_template('interaction/specialist.html', form=form) return render_template('interaction/specialist.html', form=form)
return render_template('interaction/specialists.html', form=form) return render_template('interaction/specialist.html', form=form)
@interaction_bp.route('/specialist/<int:specialist_id>', methods=['GET', 'POST']) @interaction_bp.route('/specialist/<int:specialist_id>', methods=['GET', 'POST'])
@@ -187,35 +178,31 @@ def edit_specialist(specialist_id):
form.add_dynamic_fields("configuration", configuration_config, specialist.configuration) form.add_dynamic_fields("configuration", configuration_config, specialist.configuration)
if request.method == 'GET': if request.method == 'GET':
# Pre-populate the retrievers field with current associations
current_app.logger.debug(f'Specialist retrievers: {specialist.retrievers}')
current_app.logger.debug(f'Form Retrievers Data Before: {form.retrievers.data}')
# Get the actual Retriever objects for the associated retriever_ids # Get the actual Retriever objects for the associated retriever_ids
retriever_objects = Retriever.query.filter( retriever_objects = Retriever.query.filter(
Retriever.id.in_([sr.retriever_id for sr in specialist.retrievers]) Retriever.id.in_([sr.retriever_id for sr in specialist.retrievers])
).all() ).all()
form.retrievers.data = retriever_objects form.retrievers.data = retriever_objects
current_app.logger.debug(f'Form Retrievers Data After: {form.retrievers.data}')
if form.validate_on_submit(): if form.validate_on_submit():
# Update the basic fields # Update the basic fields
form.populate_obj(specialist) specialist.name = form.name.data
specialist.description = form.description.data
specialist.tuning = form.tuning.data
# Update the configuration dynamic fields # Update the configuration dynamic fields
specialist.configuration = form.get_dynamic_data("configuration") specialist.configuration = form.get_dynamic_data("configuration")
# Update retriever associations # Get current and selected retrievers
current_retrievers = set(sr.retriever_id for sr in specialist.retrievers) current_retrievers = {sr.retriever_id: sr for sr in specialist.retrievers}
selected_retrievers = set(r.id for r in form.retrievers.data) selected_retrievers = {r.id: r for r in form.retrievers.data}
# Remove unselected retrievers # Remove unselected retrievers
for sr in specialist.retrievers[:]: for retriever_id in set(current_retrievers.keys()) - set(selected_retrievers.keys()):
if sr.retriever_id not in selected_retrievers: specialist_retriever = current_retrievers[retriever_id]
db.session.delete(sr) db.session.delete(specialist_retriever)
# Add new retrievers # Add new retrievers
for retriever_id in selected_retrievers - current_retrievers: for retriever_id in set(selected_retrievers.keys()) - set(current_retrievers.keys()):
specialist_retriever = SpecialistRetriever( specialist_retriever = SpecialistRetriever(
specialist_id=specialist.id, specialist_id=specialist.id,
retriever_id=retriever_id retriever_id=retriever_id
@@ -229,13 +216,12 @@ def edit_specialist(specialist_id):
db.session.commit() db.session.commit()
flash('Specialist updated successfully!', 'success') flash('Specialist updated successfully!', 'success')
current_app.logger.info(f'Specialist {specialist.id} updated successfully') current_app.logger.info(f'Specialist {specialist.id} updated successfully')
return redirect(prefixed_url_for('interaction_bp.specialists'))
except SQLAlchemyError as e: except SQLAlchemyError as e:
db.session.rollback() db.session.rollback()
flash(f'Failed to update specialist. Error: {str(e)}', 'danger') flash(f'Failed to update specialist. Error: {str(e)}', 'danger')
current_app.logger.error(f'Failed to update specialist {specialist_id}. Error: {str(e)}') current_app.logger.error(f'Failed to update specialist {specialist_id}. Error: {str(e)}')
return render_template('interaction/edit_specialist.html', form=form, specialist_id=specialist_id) return render_template('interaction/edit_specialist.html', form=form, specialist_id=specialist_id)
return redirect(prefixed_url_for('interaction_bp.specialists'))
else: else:
form_validation_failed(request, form) form_validation_failed(request, form)

31
eveai_app/views/security_views.py
View File

@@ -22,20 +22,11 @@ security_bp = Blueprint('security_bp', __name__)
@security_bp.before_request @security_bp.before_request
def log_before_request(): def log_before_request():
current_app.logger.debug(f"Before request (security_bp): {request.method} {request.url}") pass
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (security_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (security_bp): No user logged in")
@security_bp.after_request @security_bp.after_request
def log_after_request(response): def log_after_request(response):
current_app.logger.debug(f"After request (security_bp): {request.method} {request.url} - Status: {response.status}")
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (security_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (security_bp): No user logged in")
return response return response
@@ -47,13 +38,12 @@ def login():
form = LoginForm() form = LoginForm()
if request.method == 'POST': if request.method == 'POST':
current_app.logger.debug(f"Starting login procedure for {form.email.data}")
try: try:
if form.validate_on_submit(): if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data).first() user = User.query.filter_by(email=form.email.data).first()
if user is None or not verify_and_update_password(form.password.data, user): if user is None or not verify_and_update_password(form.password.data, user):
flash('Invalid username or password', 'danger') flash('Invalid username or password', 'danger')
current_app.logger.debug(f'Failed to login user') current_app.logger.error(f'Failed to login user')
return redirect(prefixed_url_for('security_bp.login')) return redirect(prefixed_url_for('security_bp.login'))
if login_user(user): if login_user(user):
@@ -65,10 +55,10 @@ def login():
return redirect(prefixed_url_for('user_bp.tenant_overview')) return redirect(prefixed_url_for('user_bp.tenant_overview'))
else: else:
flash('Invalid username or password', 'danger') flash('Invalid username or password', 'danger')
current_app.logger.debug(f'Failed to login user {user.email}') current_app.logger.error(f'Failed to login user {user.email}')
abort(401) abort(401)
else: else:
current_app.logger.debug(f'Invalid login form: {form.errors}') current_app.logger.error(f'Invalid login form: {form.errors}')
except CSRFError: except CSRFError:
current_app.logger.warning('CSRF token mismatch during login attempt') current_app.logger.warning('CSRF token mismatch during login attempt')
@@ -77,19 +67,14 @@ def login():
if request.method == 'GET': if request.method == 'GET':
csrf_token = generate_csrf() csrf_token = generate_csrf()
current_app.logger.debug(f'Generated new CSRF token: {csrf_token}')
# current_app.logger.debug(f"Login route completed - Session ID: {session.sid}")
current_app.logger.debug(f"Login route completed - Session data: {session}")
return render_template('security/login_user.html', login_user_form=form) return render_template('security/login_user.html', login_user_form=form)
@security_bp.route('/logout', methods=['GET', 'POST']) @security_bp.route('/logout', methods=['GET', 'POST'])
@login_required @login_required
def logout(): def logout():
current_app.logger.debug('Logging out')
logout_user() logout_user()
current_app.logger.debug('After Logout')
return redirect(prefixed_url_for('basic_bp.index')) return redirect(prefixed_url_for('basic_bp.index'))
@@ -99,17 +84,13 @@ def confirm_email(token):
email = confirm_token(token) email = confirm_token(token)
except Exception as e: except Exception as e:
flash('The confirmation link is invalid or has expired.', 'danger') flash('The confirmation link is invalid or has expired.', 'danger')
current_app.logger.debug(f'Invalid confirmation link detected: {token} - error: {e}')
return redirect(prefixed_url_for('basic_bp.confirm_email_fail')) return redirect(prefixed_url_for('basic_bp.confirm_email_fail'))
user = User.query.filter_by(email=email).first_or_404() user = User.query.filter_by(email=email).first_or_404()
current_app.logger.debug(f'Trying to confirm email for user {user.email}')
if user.active: if user.active:
flash('Account already confirmed. Please login.', 'success') flash('Account already confirmed. Please login.', 'success')
current_app.logger.debug(f'Account for user {user.email} was already activated')
return redirect(prefixed_url_for('security_bp.login')) return redirect(prefixed_url_for('security_bp.login'))
else: else:
current_app.logger.debug(f'Trying to confirm email for user {user.email}')
user.active = True user.active = True
user.updated_at = dt.now(tz.utc) user.updated_at = dt.now(tz.utc)
user.confirmed_at = dt.now(tz.utc) user.confirmed_at = dt.now(tz.utc)
@@ -119,10 +100,8 @@ def confirm_email(token):
db.session.commit() db.session.commit()
except SQLAlchemyError as e: except SQLAlchemyError as e:
db.session.rollback() db.session.rollback()
current_app.logger.debug(f'Failed to confirm email for user {user.email}: {e}')
return redirect(prefixed_url_for('basic_bp.confirm_email_fail')) return redirect(prefixed_url_for('basic_bp.confirm_email_fail'))
current_app.logger.debug(f'Account for user {user.email} was confirmed.')
send_reset_email(user) send_reset_email(user)
return redirect(prefixed_url_for('basic_bp.confirm_email_ok')) return redirect(prefixed_url_for('basic_bp.confirm_email_ok'))
@@ -145,7 +124,7 @@ def reset_password(token):
email = confirm_token(token) email = confirm_token(token)
except Exception as e: except Exception as e:
flash('The reset link is invalid or has expired.', 'danger') flash('The reset link is invalid or has expired.', 'danger')
current_app.logger.debug(f'Invalid reset link detected: {token} - error: {e}') current_app.logger.error(f'Invalid reset link detected: {token} - error: {e}')
return redirect(prefixed_url_for('security_bp.reset_password_request')) return redirect(prefixed_url_for('security_bp.reset_password_request'))
user = User.query.filter_by(email=email).first_or_404() user = User.query.filter_by(email=email).first_or_404()

20
eveai_app/views/user_views.py
View File

@@ -21,20 +21,11 @@ user_bp = Blueprint('user_bp', __name__, url_prefix='/user')
@user_bp.before_request @user_bp.before_request
def log_before_request(): def log_before_request():
current_app.logger.debug(f"Before request (user_bp): {request.method} {request.url}") pass
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (user_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (user_bp): No user logged in")
@user_bp.after_request @user_bp.after_request
def log_after_request(response): def log_after_request(response):
current_app.logger.debug(f"After request (user_bp): {request.method} {request.url} - Status: {response.status}")
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (user_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (user_bp): No user logged in")
return response return response
@@ -43,7 +34,6 @@ def log_after_request(response):
def tenant(): def tenant():
form = TenantForm() form = TenantForm()
if form.validate_on_submit(): if form.validate_on_submit():
current_app.logger.debug('Creating new tenant')
# Handle the required attributes # Handle the required attributes
new_tenant = Tenant() new_tenant = Tenant()
form.populate_obj(new_tenant) form.populate_obj(new_tenant)
@@ -91,7 +81,6 @@ def edit_tenant(tenant_id):
form.populate_obj(tenant) form.populate_obj(tenant)
if form.validate_on_submit(): if form.validate_on_submit():
current_app.logger.debug(f'Updating tenant {tenant_id}')
# Populate the tenant with form data # Populate the tenant with form data
form.populate_obj(tenant) form.populate_obj(tenant)
@@ -102,7 +91,6 @@ def edit_tenant(tenant_id):
session['tenant'] = tenant.to_dict() session['tenant'] = tenant.to_dict()
# return redirect(url_for(f"user/tenant/tenant_id")) # return redirect(url_for(f"user/tenant/tenant_id"))
else: else:
current_app.logger.debug(f'Tenant update failed with errors: {form.errors}')
form_validation_failed(request, form) form_validation_failed(request, form)
return render_template('user/tenant.html', form=form, tenant_id=tenant_id) return render_template('user/tenant.html', form=form, tenant_id=tenant_id)
@@ -142,7 +130,7 @@ def user():
# security.datastore.set_uniquifier(new_user) # security.datastore.set_uniquifier(new_user)
try: try:
send_confirmation_email(new_user) send_confirmation_email(new_user)
current_app.logger.debug(f'User {new_user.id} with name {new_user.user_name} added to database' current_app.logger.info(f'User {new_user.id} with name {new_user.user_name} added to database'
f'Confirmation email sent to {new_user.email}') f'Confirmation email sent to {new_user.email}')
flash('User added successfully and confirmation email sent.', 'success') flash('User added successfully and confirmation email sent.', 'success')
except Exception as e: except Exception as e:
@@ -448,11 +436,7 @@ def generate_api_api_key():
@user_bp.route('/tenant_overview', methods=['GET']) @user_bp.route('/tenant_overview', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin') @roles_accepted('Super User', 'Tenant Admin')
def tenant_overview(): def tenant_overview():
current_app.logger.debug('Rendering tenant overview')
current_app.logger.debug(f'current_user: {current_user}')
current_app.logger.debug(f'Current user roles: {current_user.roles}')
tenant_id = session['tenant']['id'] tenant_id = session['tenant']['id']
current_app.logger.debug(f'Generating overview for tenant {tenant_id}')
tenant = Tenant.query.get_or_404(tenant_id) tenant = Tenant.query.get_or_404(tenant_id)
form = TenantForm(obj=tenant) form = TenantForm(obj=tenant)
return render_template('user/tenant_overview.html', form=form) return render_template('user/tenant_overview.html', form=form)

37
eveai_chat/socket_handlers/chat_handler.py
View File

@@ -38,7 +38,6 @@ def track_socketio_event(func):
@track_socketio_event @track_socketio_event
def handle_connect(): def handle_connect():
try: try:
current_app.logger.debug(f'SocketIO: Connection handling started using {request.args}')
tenant_id = request.args.get('tenantId') tenant_id = request.args.get('tenantId')
if not tenant_id: if not tenant_id:
raise Exception("Missing Tenant ID") raise Exception("Missing Tenant ID")
@@ -52,12 +51,10 @@ def handle_connect():
# Create JWT token # Create JWT token
token = create_access_token(identity={"tenant_id": tenant_id, "api_key": api_key}) token = create_access_token(identity={"tenant_id": tenant_id, "api_key": api_key})
current_app.logger.debug(f'SocketIO: Connection handling created token: {token} for tenant {tenant_id}')
# Create a unique room for this client # Create a unique room for this client
room = f"{tenant_id}_{request.sid}" room = f"{tenant_id}_{request.sid}"
join_room(room) join_room(room)
current_app.logger.debug(f'SocketIO: Client joined room: {room}')
# Create a unique session ID # Create a unique session ID
if 'session_id' not in session: if 'session_id' not in session:
@@ -67,11 +64,8 @@ def handle_connect():
session['room'] = room session['room'] = room
# Communicate connection to client # Communicate connection to client
current_app.logger.debug(f'SocketIO: Connection handling sending status to client for tenant {tenant_id}')
emit('connect', {'status': 'Connected', 'tenant_id': tenant_id, 'room': room}) emit('connect', {'status': 'Connected', 'tenant_id': tenant_id, 'room': room})
current_app.logger.debug(f'SocketIO: Connection handling sending authentication token to client')
emit('authenticated', {'token': token, 'room': room}) # Emit custom event with the token emit('authenticated', {'token': token, 'room': room}) # Emit custom event with the token
current_app.logger.debug(f'SocketIO: Connection handling sent token to client for tenant {tenant_id}')
except Exception as e: except Exception as e:
current_app.logger.error(f'SocketIO: Connection failed: {e}') current_app.logger.error(f'SocketIO: Connection failed: {e}')
# communicate connection problem to client # communicate connection problem to client
@@ -85,71 +79,60 @@ def handle_disconnect():
room = session.get('room') room = session.get('room')
if room: if room:
leave_room(room) leave_room(room)
current_app.logger.debug(f'SocketIO: Client left room: {room}')
current_app.logger.debug('SocketIO: Client disconnected')
@socketio.on('heartbeat') @socketio.on('heartbeat')
def handle_heartbeat(): def handle_heartbeat():
current_app.logger.debug('SocketIO: Heartbeat received')
last_activity = session.get('last_activity') last_activity = session.get('last_activity')
if datetime.now() - last_activity > current_app.config.get('SOCKETIO_MAX_IDLE_TIME'): if datetime.now() - last_activity > current_app.config.get('SOCKETIO_MAX_IDLE_TIME'):
current_app.logger.debug('SocketIO: Heartbeat timed out, connection closed')
disconnect() disconnect()
@socketio.on('user_message') @socketio.on('user_message')
def handle_message(data): def handle_message(data):
try: try:
current_app.logger.debug(f"SocketIO: Message handling received message from tenant {data['tenantId']}: "
f"{data['message']} with token {data['token']}")
session['last_activity'] = datetime.now() session['last_activity'] = datetime.now()
current_tenant_id = validate_incoming_data(data) current_tenant_id = validate_incoming_data(data)
room = session.get('room') room = session.get('room')
# Offload actual processing of question # Offload actual processing of question
task = current_celery.send_task('ask_question', task = current_celery.send_task('execute_specialist',
queue='llm_interactions', queue='llm_interactions',
args=[ args=[
current_tenant_id, current_tenant_id,
data['message'], data['specialistId'],
data['language'], data['arguments'],
session['session_id'], session['session_id'],
data['timezone'], data['timezone'],
room room
]) ])
current_app.logger.debug(f'SocketIO: Message offloading for tenant {current_tenant_id}, '
f'Question: {task.id}')
response = { response = {
'tenantId': data['tenantId'], 'tenantId': data['tenantId'],
'message': f'Processing question ... Session ID = {session["session_id"]}', 'message': f'Processing question ... Session ID = {session["session_id"]}',
'taskId': task.id, 'taskId': task.id,
} }
current_app.logger.debug(f"SocketIO: Message handling sent bot response: {response}") current_app.logger.debug(f"Sent message with {data}, response {response}")
emit('bot_response', response, room=room) emit('bot_response', response, room=room)
except Exception as e: except Exception as e:
current_app.logger.error(f'SocketIO: Message handling failed: {e}') current_app.logger.error(f'SocketIO: Message handling failed: {str(e)}')
disconnect() disconnect()
@socketio.on('check_task_status') @socketio.on('check_task_status')
def check_task_status(data): def check_task_status(data):
current_app.logger.debug(f'SocketIO: Checking Task Status ... {data}')
task_id = data.get('task_id') task_id = data.get('task_id')
room = session.get('room') room = session.get('room')
current_app.logger.debug(f'SocketIO: Check task status for task_id: {task_id}')
if not task_id: if not task_id:
emit('task_status', {'status': 'error', 'message': 'Missing task ID'}, room=room) emit('task_status', {'status': 'error', 'message': 'Missing task ID'}, room=room)
return return
task_result = current_celery.AsyncResult(task_id) task_result = current_celery.AsyncResult(task_id)
if task_result.state == 'PENDING': if task_result.state == 'PENDING':
current_app.logger.debug(f'SocketIO: Task {task_id} is pending')
emit('task_status', {'status': 'pending', 'taskId': task_id}, room=room) emit('task_status', {'status': 'pending', 'taskId': task_id}, room=room)
elif task_result.state == 'SUCCESS': elif task_result.state == 'SUCCESS':
current_app.logger.debug(f'SocketIO: Task {task_id} has finished. Status: {task_result.state}, '
f'Result: {task_result.result}')
result = task_result.result result = task_result.result
current_app.logger.debug(f'SocketIO: Task {task_id} returned: {result}')
response = { response = {
'status': 'success', 'status': 'success',
'taskId': task_id, 'taskId': task_id,
@@ -167,8 +150,6 @@ def check_task_status(data):
@socketio.on('feedback') @socketio.on('feedback')
def handle_feedback(data): def handle_feedback(data):
try: try:
current_app.logger.debug(f'SocketIO: Feedback handling received feedback with data: {data}')
current_tenant_id = validate_incoming_data(data) current_tenant_id = validate_incoming_data(data)
interaction_id = data.get('interactionId') interaction_id = data.get('interactionId')
@@ -177,7 +158,6 @@ def handle_feedback(data):
Database(current_tenant_id).switch_schema() Database(current_tenant_id).switch_schema()
interaction = Interaction.query.get_or_404(interaction_id) interaction = Interaction.query.get_or_404(interaction_id)
current_app.logger.debug(f'Processing feedback for interaction: {interaction}')
interaction.appreciation = 0 if feedback == 'down' else 100 interaction.appreciation = 0 if feedback == 'down' else 100
try: try:
db.session.commit() db.session.commit()
@@ -188,7 +168,7 @@ def handle_feedback(data):
emit('feedback_received', {'status': 'Could not register feedback', 'interaction_id': interaction_id}) emit('feedback_received', {'status': 'Could not register feedback', 'interaction_id': interaction_id})
raise e raise e
except Exception as e: except Exception as e:
current_app.logger.debug(f'SocketIO: Feedback handling failed: {e}') current_app.logger.error(f'SocketIO: Feedback handling failed: {e}')
disconnect() disconnect()
@@ -212,7 +192,6 @@ def validate_incoming_data(data):
if not token_sub: if not token_sub:
raise Exception("Missing token subject") raise Exception("Missing token subject")
tenant_id = token_sub.get('tenant_id')
current_tenant_id = token_sub.get('tenant_id') current_tenant_id = token_sub.get('tenant_id')
if not current_tenant_id: if not current_tenant_id:

12
eveai_chat_workers/__init__.py
View File

@@ -3,11 +3,14 @@ import logging.config
from flask import Flask from flask import Flask
import os import os
from common.langchain.templates.template_manager import TemplateManager
from common.utils.celery_utils import make_celery, init_celery from common.utils.celery_utils import make_celery, init_celery
from common.extensions import db from common.extensions import db, template_manager, cache_manager
from config.logging_config import LOGGING from config.logging_config import LOGGING
from config.config import get_config from config.config import get_config
from . import specialists, retrievers
def create_app(config_file=None): def create_app(config_file=None):
app = Flask(__name__) app = Flask(__name__)
@@ -24,14 +27,12 @@ def create_app(config_file=None):
logging.config.dictConfig(LOGGING) logging.config.dictConfig(LOGGING)
app.logger.debug('Starting up eveai_chat_workers...') app.logger.info('Starting up eveai_chat_workers...')
register_extensions(app) register_extensions(app)
celery = make_celery(app.name, app.config) celery = make_celery(app.name, app.config)
init_celery(celery, app) init_celery(celery, app)
app.rag_tuning_logger = logging.getLogger('rag_tuning')
from eveai_chat_workers import tasks from eveai_chat_workers import tasks
print(tasks.tasks_ping()) print(tasks.tasks_ping())
@@ -40,6 +41,9 @@ def create_app(config_file=None):
def register_extensions(app): def register_extensions(app):
db.init_app(app) db.init_app(app)
cache_manager.init_app(app)
template_manager.init_app(app)
app, celery = create_app() app, celery = create_app()

193
eveai_chat_workers/chat_session_cache.py Normal file
View File

@@ -0,0 +1,193 @@
# common/utils/cache/chat_session_handler.py
from typing import Dict, List, Any, Optional
from datetime import datetime as dt, timezone as tz
from dataclasses import dataclass
from flask import current_app
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import joinedload
from common.extensions import db, cache_manager
from common.models.interaction import ChatSession, Interaction
from common.utils.cache.base import CacheHandler
@dataclass
class CachedInteraction:
"""Lightweight representation of an interaction for history purposes"""
specialist_arguments: Dict[str, Any] # Contains the original question and other arguments
specialist_results: Dict[str, Any] # Contains detailed question, answer and other results
@dataclass
class CachedSession:
"""Cached representation of a chat session with its interactions"""
id: int
session_id: str
interactions: List[CachedInteraction]
timezone: str
class ChatSessionCacheHandler(CacheHandler[CachedSession]):
"""Handles caching of chat sessions focused on interaction history"""
handler_name = 'chat_session_cache'
def __init__(self, region):
super().__init__(region, 'chat_session')
self.configure_keys('session_id')
def get_cached_session(self, session_id: str, *, create_params: Optional[Dict[str, Any]] = None) -> CachedSession:
"""
Get or create a cached session with its interaction history.
If not in cache, loads from database and caches it.
Args:
session_id: The session identifier
create_params: Optional parameters for session creation if it doesn't exist.
Must include 'timezone' if provided.
Returns:
CachedSession with interaction history
"""
def creator_func(session_id: str) -> CachedSession:
# Load session and interactions from database
session = (
ChatSession.query
.options(joinedload(ChatSession.interactions))
.filter_by(session_id=session_id)
.first()
)
if not session:
if not create_params:
raise ValueError(f"Chat session {session_id} not found and no creation parameters provided")
if 'timezone' not in create_params:
raise ValueError("timezone is required in create_params for new session creation")
# Create new session
session = ChatSession(
session_id=session_id,
session_start=dt.now(tz.utc),
timezone=create_params['timezone']
)
try:
db.session.add(session)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
raise ValueError(f"Failed to create new session: {str(e)}")
# Convert to cached format
cached_interactions = [
CachedInteraction(
specialist_arguments=interaction.specialist_arguments,
specialist_results=interaction.specialist_results
)
for interaction in session.interactions
if interaction.specialist_results is not None # Only include completed interactions
]
return CachedSession(
id=session.id,
session_id=session_id,
interactions=cached_interactions,
timezone=session.timezone
)
return self.get(creator_func, session_id=session_id)
def add_completed_interaction(self, session_id: str, interaction: Interaction) -> None:
"""
Add a completed interaction to the cached session history.
Should only be called once the interaction has an answer.
Args:
session_id: The session identifier
interaction: The completed interaction to add
Note:
Only adds the interaction if it has an answer
"""
if not interaction.specialist_results:
return # Skip incomplete interactions
try:
cached_session = self.get_cached_session(session_id)
# Add new interaction to cache
cached_session.interactions.append(
CachedInteraction(
specialist_arguments=interaction.specialist_arguments,
specialist_results=interaction.specialist_results,
)
)
# Force cache update
self.invalidate(session_id=session_id)
except ValueError:
# If session not in cache yet, load it fresh from DB
self.get_cached_session(session_id)
def to_cache_data(self, instance: CachedSession) -> Dict[str, Any]:
"""Convert CachedSession to cache data"""
return {
'id': instance.id,
'session_id': instance.session_id,
'timezone': instance.timezone,
'interactions': [
{
'specialist_arguments': interaction.specialist_arguments,
'specialist_results': interaction.specialist_results,
}
for interaction in instance.interactions
],
'last_updated': dt.now(tz=tz.utc).isoformat()
}
def from_cache_data(self, data: Dict[str, Any], session_id: str, **kwargs) -> CachedSession:
"""Create CachedSession from cache data"""
interactions = [
CachedInteraction(
specialist_arguments=int_data['specialist_arguments'],
specialist_results=int_data['specialist_results']
)
for int_data in data['interactions']
]
return CachedSession(
id=data['id'],
session_id=data['session_id'],
interactions=interactions,
timezone=data['timezone']
)
def should_cache(self, value: Dict[str, Any]) -> bool:
"""Validate cache data"""
required_fields = {'id','session_id', 'timezone', 'interactions'}
return all(field in value for field in required_fields)
# Register the handler with the cache manager
cache_manager.register_handler(ChatSessionCacheHandler, 'eveai_chat_workers')
# Helper function similar to get_model_variables
def get_chat_history(session_id: str) -> CachedSession:
"""
Get cached chat history for a session, loading from database if needed
Args:
session_id: Session ID to look up
Returns:
CachedSession with interaction history
Raises:
ValueError: If session doesn't exist
"""
return cache_manager.chat_session_cache.get_cached_session(session_id)

5
eveai_chat_workers/retrievers/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
# Import all specialist implementations here to ensure registration
from . import standard_rag
# List of all available specialist implementations
__all__ = ['standard_rag']

57
eveai_chat_workers/retrievers/base.py Normal file
View File

@@ -0,0 +1,57 @@
from abc import ABC, abstractmethod, abstractproperty
from typing import Dict, Any, List
from flask import current_app
from eveai_chat_workers.retrievers.retriever_typing import RetrieverResult, RetrieverArguments
from config.logging_config import TuningLogger
class BaseRetriever(ABC):
"""Base class for all retrievers"""
def __init__(self, tenant_id: int, retriever_id: int):
self.tenant_id = tenant_id
self.retriever_id = retriever_id
self.tuning = False
self.tuning_logger = None
self._setup_tuning_logger()
@property
@abstractmethod
def type(self) -> str:
"""The type of the retriever"""
pass
def _setup_tuning_logger(self):
try:
self.tuning_logger = TuningLogger(
'tuning',
tenant_id=self.tenant_id,
retriever_id=self.retriever_id,
)
# Verify logger is working with a test message
if self.tuning:
self.tuning_logger.log_tuning('retriever', "Tuning logger initialized")
except Exception as e:
current_app.logger.error(f"Failed to setup tuning logger: {str(e)}")
raise
def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None:
if self.tuning and self.tuning_logger:
try:
self.tuning_logger.log_tuning('retriever', message, data)
except Exception as e:
current_app.logger.error(f"Processor: Error in tuning logging: {e}")
@abstractmethod
def retrieve(self, arguments: RetrieverArguments) -> List[RetrieverResult]:
"""
Retrieve relevant documents based on provided arguments
Args:
arguments: Dictionary of arguments for the retrieval operation
Returns:
List[Dict[str, Any]]: List of retrieved documents/content
"""
pass

20
eveai_chat_workers/retrievers/registry.py Normal file
View File

@@ -0,0 +1,20 @@
from typing import Dict, Type
from .base import BaseRetriever
class RetrieverRegistry:
"""Registry for retriever types"""
_registry: Dict[str, Type[BaseRetriever]] = {}
@classmethod
def register(cls, retriever_type: str, retriever_class: Type[BaseRetriever]):
"""Register a new retriever type"""
cls._registry[retriever_type] = retriever_class
@classmethod
def get_retriever_class(cls, retriever_type: str) -> Type[BaseRetriever]:
"""Get the retriever class for a given type"""
if retriever_type not in cls._registry:
raise ValueError(f"Unknown retriever type: {retriever_type}")
return cls._registry[retriever_type]

60
eveai_chat_workers/retrievers/retriever_typing.py Normal file
View File

@@ -0,0 +1,60 @@
from typing import List, Dict, Any
from pydantic import BaseModel, Field, model_validator
from common.utils.config_field_types import ArgumentDefinition, TaggingFields
from config.retriever_types import RETRIEVER_TYPES
class RetrieverMetadata(BaseModel):
"""Metadata structure for retrieved documents"""
document_id: int = Field(..., description="ID of the source document")
version_id: int = Field(..., description="Version ID of the source document")
document_name: str = Field(..., description="Name of the source document")
user_metadata: Dict[str, Any] = Field(
default_factory=dict, # This will use an empty dict if None is provided
description="User-defined metadata"
)
class RetrieverResult(BaseModel):
"""Standard result format for all retrievers"""
id: int = Field(..., description="ID of the retrieved embedding")
chunk: str = Field(..., description="Retrieved text chunk")
similarity: float = Field(..., description="Similarity score (0-1)")
metadata: RetrieverMetadata = Field(..., description="Associated metadata")
class RetrieverArguments(BaseModel):
"""
Dynamic arguments for retrievers, allowing arbitrary fields but validating required ones
based on RETRIEVER_TYPES configuration.
"""
type: str = Field(..., description="Type of retriever (e.g. STANDARD_RAG)")
# Allow any additional fields
model_config = {
"extra": "allow"
}
@model_validator(mode='after')
def validate_required_arguments(self) -> 'RetrieverArguments':
"""Validate that all required arguments for this retriever type are present"""
retriever_config = RETRIEVER_TYPES.get(self.type)
if not retriever_config:
raise ValueError(f"Unknown retriever type: {self.type}")
# Check required arguments from configuration
for arg_name, arg_config in retriever_config['arguments'].items():
if arg_config.get('required', False):
if not hasattr(self, arg_name):
raise ValueError(f"Missing required argument '{arg_name}' for {self.type}")
# Type validation
value = getattr(self, arg_name)
expected_type = arg_config['type']
if expected_type == 'str' and not isinstance(value, str):
raise ValueError(f"Argument '{arg_name}' must be a string")
elif expected_type == 'int' and not isinstance(value, int):
raise ValueError(f"Argument '{arg_name}' must be an integer")
# Add other type validations as needed
return self

140
eveai_chat_workers/retrievers/standard_rag.py Normal file
View File

@@ -0,0 +1,140 @@
# retrievers/standard_rag.py
from datetime import datetime as dt, timezone as tz
from typing import Dict, Any, List
from sqlalchemy import func, or_, desc
from sqlalchemy.exc import SQLAlchemyError
from flask import current_app
from common.extensions import db
from common.models.document import Document, DocumentVersion, Catalog, Retriever
from common.models.user import Tenant
from common.utils.datetime_utils import get_date_in_timezone
from common.utils.model_utils import get_model_variables
from .base import BaseRetriever
from .registry import RetrieverRegistry
from .retriever_typing import RetrieverArguments, RetrieverResult, RetrieverMetadata
class StandardRAGRetriever(BaseRetriever):
"""Standard RAG retriever implementation"""
def __init__(self, tenant_id: int, retriever_id: int):
super().__init__(tenant_id, retriever_id)
retriever = Retriever.query.get_or_404(retriever_id)
self.catalog_id = retriever.catalog_id
self.similarity_threshold = retriever.configuration.get('es_similarity_threshold', 0.3)
self.k = retriever.configuration.get('es_k', 8)
self.tuning = retriever.tuning
self.model_variables = get_model_variables(self.tenant_id)
self._log_tuning("Standard RAG retriever initialized")
@property
def type(self) -> str:
return "STANDARD_RAG"
def retrieve(self, arguments: RetrieverArguments) -> List[RetrieverResult]:
"""
Retrieve documents based on query
Args:
arguments: Validated RetrieverArguments containing at minimum:
- query: str - The search query
Returns:
List[RetrieverResult]: List of retrieved documents with similarity scores
"""
try:
query = arguments.query
# Get query embedding
query_embedding = self._get_query_embedding(query)
# Get the appropriate embedding database model
db_class = self.model_variables.embedding_model_class
# Get current date for validity checks
current_date = dt.now(tz=tz.utc).date()
# Create subquery for latest versions
subquery = (
db.session.query(
DocumentVersion.doc_id,
func.max(DocumentVersion.id).label('latest_version_id')
)
.group_by(DocumentVersion.doc_id)
.subquery()
)
# Main query
query_obj = (
db.session.query(
db_class,
(1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity')
)
.join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
.join(Document, DocumentVersion.doc_id == Document.id)
.join(subquery, DocumentVersion.id == subquery.c.latest_version_id)
.filter(
or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date),
(1 - db_class.embedding.cosine_distance(query_embedding)) > self.similarity_threshold,
Document.catalog_id == self.catalog_id
)
.order_by(desc('similarity'))
.limit(self.k)
)
results = query_obj.all()
# Transform results into standard format
processed_results = []
for doc, similarity in results:
processed_results.append(
RetrieverResult(
id=doc.id,
chunk=doc.chunk,
similarity=float(similarity),
metadata=RetrieverMetadata(
document_id=doc.document_version.doc_id,
version_id=doc.document_version.id,
document_name=doc.document_version.document.name,
user_metadata=doc.document_version.user_metadata or {},
)
)
)
# Log the retrieval
if self.tuning:
compiled_query = str(query_obj.statement.compile(
compile_kwargs={"literal_binds": True} # This will include the actual values in the SQL
))
self._log_tuning('retrieve', {
"arguments": arguments.model_dump(),
"similarity_threshold": self.similarity_threshold,
"k": self.k,
"query": compiled_query,
"Raw Results": str(results),
"Processed Results": [r.model_dump() for r in processed_results],
})
return processed_results
except SQLAlchemyError as e:
current_app.logger.error(f'Error in RAG retrieval: {e}')
db.session.rollback()
raise
except Exception as e:
current_app.logger.error(f'Unexpected error in RAG retrieval: {e}')
raise
def _get_query_embedding(self, query: str):
"""Get embedding for the query text"""
embedding_model = self.model_variables.embedding_model
return embedding_model.embed_query(query)
# Register the retriever type
RetrieverRegistry.register("STANDARD_RAG", StandardRAGRetriever)

5
eveai_chat_workers/specialists/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
# Import all specialist implementations here to ensure registration
from . import rag_specialist
# List of all available specialist implementations
__all__ = ['rag_specialist']

50
eveai_chat_workers/specialists/base.py Normal file
View File

@@ -0,0 +1,50 @@
from abc import ABC, abstractmethod
from typing import Dict, Any
from flask import current_app
from config.logging_config import TuningLogger
from eveai_chat_workers.specialists.specialist_typing import SpecialistArguments, SpecialistResult
class BaseSpecialist(ABC):
"""Base class for all specialists"""
def __init__(self, tenant_id: int, specialist_id: int, session_id: str):
self.tenant_id = tenant_id
self.specialist_id = specialist_id
self.session_id = session_id
self.tuning = False
self.tuning_logger = None
self._setup_tuning_logger()
@property
@abstractmethod
def type(self) -> str:
"""The type of the specialist"""
pass
def _setup_tuning_logger(self):
try:
self.tuning_logger = TuningLogger(
'tuning',
tenant_id=self.tenant_id,
specialist_id=self.specialist_id,
)
# Verify logger is working with a test message
if self.tuning:
self.tuning_logger.log_tuning('specialist', "Tuning logger initialized")
except Exception as e:
current_app.logger.error(f"Failed to setup tuning logger: {str(e)}")
raise
def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None:
if self.tuning and self.tuning_logger:
try:
self.tuning_logger.log_tuning('specialist', message, data)
except Exception as e:
current_app.logger.error(f"Processor: Error in tuning logging: {e}")
@abstractmethod
def execute(self, arguments: SpecialistArguments) -> SpecialistResult:
"""Execute the specialist's logic"""
pass

289
eveai_chat_workers/specialists/rag_specialist.py Normal file
View File

@@ -0,0 +1,289 @@
from datetime import datetime
from typing import Dict, Any, List
from flask import current_app
from langchain_core.exceptions import LangChainException
from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnableParallel, RunnablePassthrough
from common.langchain.outputs.base import OutputRegistry
from common.langchain.outputs.rag import RAGOutput
from common.utils.business_event_context import current_event
from .specialist_typing import SpecialistArguments, SpecialistResult
from ..chat_session_cache import CachedInteraction, get_chat_history
from ..retrievers.registry import RetrieverRegistry
from ..retrievers.base import BaseRetriever
from common.models.interaction import SpecialistRetriever, Specialist
from common.utils.model_utils import get_model_variables, create_language_template, replace_variable_in_template
from .base import BaseSpecialist
from .registry import SpecialistRegistry
from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments, RetrieverResult
class RAGSpecialist(BaseSpecialist):
"""
Standard Q&A RAG Specialist implementation that combines retriever results
with LLM processing to generate answers.
"""
def __init__(self, tenant_id: int, specialist_id: int, session_id: str):
super().__init__(tenant_id, specialist_id, session_id)
# Check and load the specialist
specialist = Specialist.query.get_or_404(specialist_id)
# Set the specific configuration for the RAG Specialist
self.specialist_context = specialist.configuration.get('specialist_context', '')
self.temperature = specialist.configuration.get('temperature', 0.3)
self.tuning = specialist.tuning
# Initialize retrievers
self.retrievers = self._initialize_retrievers()
# Initialize model variables
self.model_variables = get_model_variables(tenant_id)
@property
def type(self) -> str:
return "STANDARD_RAG"
def _initialize_retrievers(self) -> List[BaseRetriever]:
"""Initialize all retrievers associated with this specialist"""
retrievers = []
# Get retriever associations from database
specialist_retrievers = (
SpecialistRetriever.query
.filter_by(specialist_id=self.specialist_id)
.all()
)
self._log_tuning("_initialize_retrievers", {"Nr of retrievers": len(specialist_retrievers)})
for spec_retriever in specialist_retrievers:
# Get retriever configuration from database
retriever = spec_retriever.retriever
retriever_class = RetrieverRegistry.get_retriever_class(retriever.type)
self._log_tuning("_initialize_retrievers", {
"Retriever id": spec_retriever.retriever_id,
"Retriever Type": retriever.type,
"Retriever Class": str(retriever_class),
})
# Initialize retriever with its configuration
retrievers.append(
retriever_class(
tenant_id=self.tenant_id,
retriever_id=retriever.id,
)
)
return retrievers
@property
def required_templates(self) -> List[str]:
"""List of required templates for this specialist"""
return ['rag', 'history']
# def _detail_question(question, language, model_variables, session_id):
# retriever = EveAIHistoryRetriever(model_variables=model_variables, session_id=session_id)
# llm = model_variables['llm']
# template = model_variables['history_template']
# language_template = create_language_template(template, language)
# full_template = replace_variable_in_template(language_template, "{tenant_context}",
# model_variables['rag_context'])
# history_prompt = ChatPromptTemplate.from_template(full_template)
# setup_and_retrieval = RunnableParallel({"history": retriever, "question": RunnablePassthrough()})
# output_parser = StrOutputParser()
#
# chain = setup_and_retrieval | history_prompt | llm | output_parser
#
# try:
# answer = chain.invoke(question)
# return answer
# except LangChainException as e:
# current_app.logger.error(f'Error detailing question: {e}')
# raise
def _detail_question(self, language: str, question: str) -> str:
"""Detail question based on conversation history"""
try:
# Get cached session history
cached_session = get_chat_history(self.session_id)
# Format history for the prompt
formatted_history = "\n\n".join([
f"HUMAN:\n{interaction.specialist_results.get('detailed_query')}\n\n"
f"AI:\n{interaction.specialist_results.get('answer')}"
for interaction in cached_session.interactions
])
# Get LLM and template
llm = self.model_variables.get_llm(temperature=0.3)
template = self.model_variables.get_template('history')
language_template = create_language_template(template, language)
# Create prompt
history_prompt = ChatPromptTemplate.from_template(language_template)
# Create chain
chain = (
history_prompt |
llm |
StrOutputParser()
)
# Execute chain
detailed_question = chain.invoke({
"history": formatted_history,
"question": question
})
if self.tuning:
self._log_tuning("_detail_question", {
"cached_session_id": cached_session.session_id,
"cached_session.interactions": str(cached_session.interactions),
"original_question": question,
"history_used": formatted_history,
"detailed_question": detailed_question,
})
return detailed_question
except Exception as e:
current_app.logger.error(f"Error detailing question: {e}")
return question # Fallback to original question
def execute(self, arguments: SpecialistArguments) -> SpecialistResult:
"""
Execute the RAG specialist to generate an answer
"""
start_time = datetime.now()
try:
with current_event.create_span("Specialist Detail Question"):
# Get required arguments
language = arguments.language
query = arguments.query
detailed_question = self._detail_question(language, query)
# Log the start of retrieval process if tuning is enabled
with current_event.create_span("Specialist Retrieval"):
self._log_tuning("Starting context retrieval", {
"num_retrievers": len(self.retrievers),
"all arguments": arguments.model_dump(),
})
# Get retriever-specific arguments
retriever_arguments = arguments.retriever_arguments
# Collect context from all retrievers
all_context = []
for retriever in self.retrievers:
# Get arguments for this specific retriever
retriever_id = str(retriever.retriever_id)
if retriever_id not in retriever_arguments:
current_app.logger.error(f"Missing arguments for retriever {retriever_id}")
continue
# Get the retriever's arguments and update the query
current_retriever_args = retriever_arguments[retriever_id]
if isinstance(retriever_arguments[retriever_id], RetrieverArguments):
updated_args = current_retriever_args.model_dump()
updated_args['query'] = detailed_question
retriever_args = RetrieverArguments(**updated_args)
else:
# Create a new RetrieverArguments instance from the dictionary
current_retriever_args['query'] = detailed_question
retriever_args = RetrieverArguments(**current_retriever_args)
# Each retriever gets its own specific arguments
retriever_result = retriever.retrieve(retriever_args)
all_context.extend(retriever_result)
# Sort by similarity if available and get unique contexts
all_context.sort(key=lambda x: x.similarity, reverse=True)
unique_contexts = []
seen_chunks = set()
for ctx in all_context:
if ctx.chunk not in seen_chunks:
unique_contexts.append(ctx)
seen_chunks.add(ctx.chunk)
self._log_tuning("Context retrieval completed", {
"total_contexts": len(all_context),
"unique_contexts": len(unique_contexts),
"average_similarity": sum(ctx.similarity for ctx in unique_contexts) / len(
unique_contexts) if unique_contexts else 0
})
# Prepare context for LLM
formatted_context = "\n\n".join([
f"SOURCE: {ctx.metadata.document_id}\n\n{ctx.chunk}"
for ctx in unique_contexts
])
with current_event.create_span("Specialist RAG invocation"):
try:
# Get LLM with specified temperature
llm = self.model_variables.get_llm(temperature=self.temperature)
# Get template
template = self.model_variables.get_template('rag')
language_template = create_language_template(template, language)
full_template = replace_variable_in_template(
language_template,
"{tenant_context}",
self.specialist_context
)
if self.tuning:
self._log_tuning("Template preparation completed", {
"template": full_template,
"context": formatted_context,
"tenant_context": self.specialist_context,
})
# Create prompt
rag_prompt = ChatPromptTemplate.from_template(full_template)
# Setup chain components
setup_and_retrieval = RunnableParallel({
"context": lambda x: formatted_context,
"question": lambda x: x
})
# Get output schema for structured output
output_schema = OutputRegistry.get_schema(self.type)
structured_llm = llm.with_structured_output(output_schema)
chain = setup_and_retrieval | rag_prompt | structured_llm
raw_result = chain.invoke(query)
result = SpecialistResult.create_for_type(
"STANDARD_RAG",
detailed_query=detailed_question,
answer=raw_result.answer,
citations=[ctx.metadata.document_id for ctx in unique_contexts
if ctx.id in raw_result.citations],
insufficient_info=raw_result.insufficient_info
)
if self.tuning:
self._log_tuning("LLM chain execution completed", {
"Result": result.model_dump()
})
except Exception as e:
current_app.logger.error(f"Error in LLM processing: {e}")
if self.tuning:
self._log_tuning("LLM processing error", {"error": str(e)})
raise
return result
except Exception as e:
current_app.logger.error(f'Error in RAG specialist execution: {str(e)}')
raise
# Register the specialist type
SpecialistRegistry.register("STANDARD_RAG", RAGSpecialist)
OutputRegistry.register("STANDARD_RAG", RAGOutput)

21
eveai_chat_workers/specialists/registry.py Normal file
View File

@@ -0,0 +1,21 @@
from typing import Dict, Type
from .base import BaseSpecialist
class SpecialistRegistry:
"""Registry for specialist types"""
_registry: Dict[str, Type[BaseSpecialist]] = {}
@classmethod
def register(cls, specialist_type: str, specialist_class: Type[BaseSpecialist]):
"""Register a new specialist type"""
cls._registry[specialist_type] = specialist_class
@classmethod
def get_specialist_class(cls, specialist_type: str) -> Type[BaseSpecialist]:
"""Get the specialist class for a given type"""
if specialist_type not in cls._registry:
raise ValueError(f"Unknown specialist type: {specialist_type}")
return cls._registry[specialist_type]

144
eveai_chat_workers/specialists/specialist_typing.py Normal file
View File

@@ -0,0 +1,144 @@
from typing import List, Dict, Any, Optional
from pydantic import BaseModel, Field, model_validator
from config.specialist_types import SPECIALIST_TYPES
from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments
class SpecialistArguments(BaseModel):
"""
Dynamic arguments for specialists, allowing arbitrary fields but validating required ones
based on SPECIALIST_TYPES configuration.
"""
type: str = Field(..., description="Type of specialist (e.g. STANDARD_RAG)")
retriever_arguments: Dict[str, Any] = Field(
default_factory=dict,
description="Arguments for each retriever, keyed by retriever ID"
)
# Allow any additional fields
model_config = {
"extra": "allow"
}
@model_validator(mode='after')
def validate_required_arguments(self) -> 'SpecialistArguments':
"""Validate that all required arguments for this specialist type are present"""
specialist_config = SPECIALIST_TYPES.get(self.type)
if not specialist_config:
raise ValueError(f"Unknown specialist type: {self.type}")
# Check required arguments from configuration
for arg_name, arg_config in specialist_config['arguments'].items():
if arg_config.get('required', False):
if not hasattr(self, arg_name):
raise ValueError(f"Missing required argument '{arg_name}' for {self.type}")
# Type validation
value = getattr(self, arg_name)
expected_type = arg_config['type']
if expected_type == 'str' and not isinstance(value, str):
raise ValueError(f"Argument '{arg_name}' must be a string")
elif expected_type == 'int' and not isinstance(value, int):
raise ValueError(f"Argument '{arg_name}' must be an integer")
return self
@classmethod
def create(cls, type_name: str, specialist_args: Dict[str, Any],
retriever_args: Dict[str, Dict[str, Any]]) -> 'SpecialistArguments':
"""
Factory method to create SpecialistArguments with validated retriever arguments
Args:
type_name: The specialist type (e.g., 'STANDARD_RAG')
specialist_args: Arguments specific to the specialist
retriever_args: Dictionary of retriever arguments keyed by retriever ID
Returns:
Validated SpecialistArguments instance
"""
# Convert raw retriever arguments to RetrieverArguments instances
validated_retriever_args = {}
for retriever_id, args in retriever_args.items():
# Ensure type is included in retriever arguments
if 'type' not in args:
raise ValueError(f"Retriever arguments for {retriever_id} must include 'type'")
validated_retriever_args[retriever_id] = RetrieverArguments(**args)
# Combine everything into the specialist arguments
return cls(
type=type_name,
**specialist_args,
retriever_arguments=validated_retriever_args
)
class SpecialistResult(BaseModel):
"""
Dynamic results from specialists, validating required fields based on
SPECIALIST_TYPES configuration.
"""
type: str = Field(..., description="Type of specialist (e.g. STANDARD_RAG)")
# Allow any additional fields
model_config = {
"extra": "allow"
}
@model_validator(mode='after')
def validate_required_results(self) -> 'SpecialistResult':
"""Validate that all required result fields for this specialist type are present"""
specialist_config = SPECIALIST_TYPES.get(self.type)
if not specialist_config:
raise ValueError(f"Unknown specialist type: {self.type}")
# Check required results from configuration
required_results = specialist_config.get('results', {})
for result_name, result_config in required_results.items():
if result_config.get('required', False):
if not hasattr(self, result_name):
raise ValueError(f"Missing required result '{result_name}' for {self.type}")
# Type validation
value = getattr(self, result_name)
expected_type = result_config['type']
# Validate based on type annotation
if expected_type == 'str' and not isinstance(value, str):
raise ValueError(f"Result '{result_name}' must be a string")
elif expected_type == 'bool' and not isinstance(value, bool):
raise ValueError(f"Result '{result_name}' must be a boolean")
elif expected_type == 'List[str]' and not (
isinstance(value, list) and all(isinstance(x, str) for x in value)):
raise ValueError(f"Result '{result_name}' must be a list of strings")
# Add other type validations as needed
return self
@classmethod
def create_for_type(cls, specialist_type: str, **results) -> 'SpecialistResult':
"""
Factory method to create a type-specific result
Args:
specialist_type: The type of specialist (e.g., 'STANDARD_RAG')
**results: The result values to include
Returns:
Validated SpecialistResult instance
Example:
For STANDARD_RAG:
result = SpecialistResult.create_for_type(
'STANDARD_RAG',
answer="The answer text",
citations=["doc1", "doc2"],
insufficient_info=False
)
"""
# Add the type to the results
results['type'] = specialist_type
# Create and validate the result
return cls(**results)

512
eveai_chat_workers/tasks.py
View File

@@ -1,24 +1,23 @@
from datetime import datetime as dt, timezone as tz from datetime import datetime as dt, timezone as tz
from typing import Dict, Any, Optional
from flask import current_app from flask import current_app
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnableParallel, RunnablePassthrough
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
# OpenAI imports from common.utils.config_field_types import TaggingFields
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.exceptions import LangChainException
from common.utils.database import Database from common.utils.database import Database
from common.models.document import Embedding from common.models.document import Embedding, Catalog
from common.models.user import Tenant from common.models.user import Tenant
from common.models.interaction import ChatSession, Interaction, InteractionEmbedding from common.models.interaction import ChatSession, Interaction, InteractionEmbedding, Specialist, SpecialistRetriever
from common.extensions import db from common.extensions import db, cache_manager
from common.utils.celery_utils import current_celery from common.utils.celery_utils import current_celery
from common.utils.model_utils import select_model_variables, create_language_template, replace_variable_in_template
from common.langchain.retrievers.eveai_default_rag_retriever import EveAIDefaultRagRetriever
from common.langchain.retrievers.eveai_history_retriever import EveAIHistoryRetriever
from common.utils.business_event import BusinessEvent from common.utils.business_event import BusinessEvent
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
from config.specialist_types import SPECIALIST_TYPES
from eveai_chat_workers.chat_session_cache import get_chat_history
from eveai_chat_workers.specialists.registry import SpecialistRegistry
from config.retriever_types import RETRIEVER_TYPES
from eveai_chat_workers.specialists.specialist_typing import SpecialistArguments
# Healthcheck task # Healthcheck task
@@ -27,41 +26,207 @@ def ping():
return 'pong' return 'pong'
def detail_question(question, language, model_variables, session_id): class ArgumentPreparationError(Exception):
current_app.logger.debug(f'Detail question: {question}') """Custom exception for argument preparation errors"""
current_app.logger.debug(f'model_variables: {model_variables}') pass
current_app.logger.debug(f'session_id: {session_id}')
retriever = EveAIHistoryRetriever(model_variables=model_variables, session_id=session_id)
llm = model_variables['llm']
template = model_variables['history_template']
language_template = create_language_template(template, language)
full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
history_prompt = ChatPromptTemplate.from_template(full_template)
setup_and_retrieval = RunnableParallel({"history": retriever, "question": RunnablePassthrough()})
output_parser = StrOutputParser()
chain = setup_and_retrieval | history_prompt | llm | output_parser
try:
answer = chain.invoke(question)
return answer
except LangChainException as e:
current_app.logger.error(f'Error detailing question: {e}')
raise
@current_celery.task(name='ask_question', queue='llm_interactions') def validate_specialist_arguments(specialist_type: str, arguments: Dict[str, Any]) -> None:
def ask_question(tenant_id, question, language, session_id, user_timezone, room):
"""returns result structured as follows:
result = {
'answer': 'Your answer here',
'citations': ['http://example.com/citation1', 'http://example.com/citation2'],
'algorithm': 'algorithm_name',
'interaction_id': 'interaction_id_value'
}
""" """
with BusinessEvent("Ask Question", tenant_id=tenant_id, chat_session_id=session_id): Validate specialist-specific arguments
current_app.logger.info(f'ask_question: Received question for tenant {tenant_id}: {question}. Processing...')
Args:
specialist_type: Type of specialist
arguments: Arguments to validate (excluding retriever-specific arguments)
Raises:
ArgumentPreparationError: If validation fails
"""
specialist_config = SPECIALIST_TYPES.get(specialist_type)
if not specialist_config:
raise ArgumentPreparationError(f"Unknown specialist type: {specialist_type}")
required_args = specialist_config.get('arguments', {})
# Check for required arguments
for arg_name, arg_config in required_args.items():
if arg_config.get('required', False) and arg_name not in arguments:
raise ArgumentPreparationError(f"Missing required argument '{arg_name}' for specialist")
if arg_name in arguments:
# Type checking
expected_type = arg_config.get('type')
if expected_type == 'str' and not isinstance(arguments[arg_name], str):
raise ArgumentPreparationError(f"Argument '{arg_name}' must be a string")
elif expected_type == 'int' and not isinstance(arguments[arg_name], int):
raise ArgumentPreparationError(f"Argument '{arg_name}' must be an integer")
def validate_retriever_arguments(retriever_type: str, arguments: Dict[str, Any],
catalog_config: Optional[Dict[str, Any]] = None) -> None:
"""
Validate retriever-specific arguments
Args:
retriever_type: Type of retriever
arguments: Arguments to validate
catalog_config: Optional catalog configuration for metadata validation
Raises:
ArgumentPreparationError: If validation fails
"""
retriever_config = RETRIEVER_TYPES.get(retriever_type)
if not retriever_config:
raise ArgumentPreparationError(f"Unknown retriever type: {retriever_type}")
# Validate standard retriever arguments
required_args = retriever_config.get('arguments', {})
for arg_name, arg_config in required_args.items():
if arg_config.get('required', False) and arg_name not in arguments:
raise ArgumentPreparationError(f"Missing required argument '{arg_name}' for retriever")
# Only validate metadata filters if catalog configuration is provided
if catalog_config and 'metadata_filters' in arguments:
if 'tagging_fields' in catalog_config:
tagging_fields = TaggingFields.from_dict(catalog_config['tagging_fields'])
errors = tagging_fields.validate_argument_values(arguments['metadata_filters'])
if errors:
raise ArgumentPreparationError(f"Invalid metadata filters: {errors}")
def is_retriever_id(key: str) -> bool:
"""
Check if a key represents a valid retriever ID.
Valid formats: positive integers, including leading zeros
Args:
key: String to check
Returns:
bool: True if the key represents a valid retriever ID
"""
try:
# Convert to int to handle leading zeros
value = int(key)
# Ensure it's a positive number
return value > 0
except ValueError:
return False
def prepare_arguments(specialist: Any, arguments: Dict[str, Any]) -> Dict[str, Any]:
"""
Prepare complete argument dictionary for specialist execution with inheritance
Args:
specialist: Specialist model instance
arguments: Dictionary containing:
- Specialist arguments
- Retriever-specific arguments keyed by retriever ID
Returns:
Dict containing prepared arguments with inheritance applied
Raises:
ArgumentPreparationError: If argument preparation or validation fails
"""
try:
# Separate specialist arguments from retriever arguments
retriever_args = {}
specialist_args = {}
for key, value in arguments.items():
if isinstance(key, str) and is_retriever_id(key): # Retriever ID
retriever_args[key] = value
else:
specialist_args[key] = value
# Validate specialist arguments
validate_specialist_arguments(specialist.type, specialist_args)
# Get all retrievers associated with this specialist
specialist_retrievers = (
SpecialistRetriever.query
.filter_by(specialist_id=specialist.id)
.all()
)
# Process each retriever
prepared_retriever_args = {}
for spec_retriever in specialist_retrievers:
retriever = spec_retriever.retriever
retriever_id = str(retriever.id)
# Get catalog configuration if it exists
catalog_config = None
if retriever.catalog_id:
try:
catalog = Catalog.query.get(retriever.catalog_id)
if catalog:
catalog_config = catalog.configuration
except SQLAlchemyError:
current_app.logger.warning(
f"Could not fetch catalog {retriever.catalog_id} for retriever {retriever_id}"
)
# Start with specialist arguments (inheritance)
inherited_args = specialist_args.copy()
# Override with retriever-specific arguments if provided
if retriever_id in retriever_args:
inherited_args.update(retriever_args[retriever_id])
# Always include the retriever type
inherited_args['type'] = retriever.type
# Validate the combined arguments
validate_retriever_arguments(
retriever.type,
inherited_args,
catalog_config
)
prepared_retriever_args[retriever_id] = inherited_args
# Construct final argument structure
final_arguments = {
**specialist_args,
'retriever_arguments': prepared_retriever_args
}
return final_arguments
except SQLAlchemyError as e:
current_app.logger.error(f'Database error during argument preparation: {e}')
raise ArgumentPreparationError(f"Database error: {str(e)}")
except Exception as e:
current_app.logger.error(f'Error during argument preparation: {e}')
raise ArgumentPreparationError(str(e))
@current_celery.task(name='execute_specialist', queue='llm_interactions')
def execute_specialist(tenant_id: int, specialist_id: int, arguments: Dict[str, Any],
session_id: str, user_timezone: str, room: str) -> dict:
"""
Execute a specialist with given arguments
Args:
tenant_id: ID of the tenant
specialist_id: ID of the specialist to use
arguments: Dictionary containing all required arguments for specialist and retrievers
session_id: Chat session ID
user_timezone: User's timezone
room: Socket.IO room for the response
Returns:
dict: {
'result': Dict - Specialist execution result
'interaction_id': int - Created interaction ID
'room': str - Socket.IO room
}
"""
with BusinessEvent("Execute Specialist", tenant_id=tenant_id, chat_session_id=session_id) as event:
current_app.logger.info(
f'execute_specialist: Processing request for tenant {tenant_id} using specialist {specialist_id}')
try: try:
# Retrieve the tenant # Retrieve the tenant
@@ -69,208 +234,85 @@ def ask_question(tenant_id, question, language, session_id, user_timezone, room)
if not tenant: if not tenant:
raise Exception(f'Tenant {tenant_id} not found.') raise Exception(f'Tenant {tenant_id} not found.')
# Ensure we are working in the correct database schema # Switch to correct database schema
Database(tenant_id).switch_schema() Database(tenant_id).switch_schema()
# Ensure we have a session to story history # Ensure we have a session
chat_session = ChatSession.query.filter_by(session_id=session_id).first() cached_session = cache_manager.chat_session_cache.get_cached_session(
if not chat_session: session_id,
create_params={'timezone': user_timezone}
)
# Get specialist from database
specialist = Specialist.query.get_or_404(specialist_id)
# Prepare complete arguments
try:
raw_arguments = prepare_arguments(specialist, arguments)
# Convert the prepared arguments into a SpecialistArguments instance
complete_arguments = SpecialistArguments.create(
type_name=specialist.type,
specialist_args={k: v for k, v in raw_arguments.items() if k != 'retriever_arguments'},
retriever_args=raw_arguments.get('retriever_arguments', {})
)
except ValueError as e:
current_app.logger.error(f'execute_specialist: Error preparing arguments: {e}')
raise
# Create new interaction record
new_interaction = Interaction()
new_interaction.chat_session_id = cached_session.id
new_interaction.timezone = user_timezone
new_interaction.question_at = dt.now(tz.utc)
new_interaction.specialist_id = specialist.id
new_interaction.specialist_arguments = complete_arguments.model_dump(mode='json')
try:
db.session.add(new_interaction)
db.session.commit()
event.update_attribute('interaction_id', new_interaction.id)
except SQLAlchemyError as e:
current_app.logger.error(f'execute_specialist: Error creating interaction: {e}')
raise
with current_event.create_span("Specialist invocation"):
# Initialize specialist instance
specialist_class = SpecialistRegistry.get_specialist_class(specialist.type)
specialist_instance = specialist_class(
tenant_id=tenant_id,
specialist_id=specialist_id,
session_id=session_id,
)
# Execute specialist
result = specialist_instance.execute(complete_arguments)
# Update interaction record
new_interaction.specialist_results = result.model_dump(mode='json') # Store complete result
new_interaction.answer_at = dt.now(tz.utc)
try: try:
chat_session = ChatSession() db.session.add(new_interaction)
chat_session.session_id = session_id
chat_session.session_start = dt.now(tz.utc)
chat_session.timezone = user_timezone
db.session.add(chat_session)
db.session.commit() db.session.commit()
except SQLAlchemyError as e: except SQLAlchemyError as e:
current_app.logger.error(f'ask_question: Error initializing chat session in database: {e}') current_app.logger.error(f'execute_specialist: Error updating interaction: {e}')
raise raise
with current_event.create_span("RAG Answer"): # Now that we have a complete interaction with an answer, add it to the cache
result, interaction = answer_using_tenant_rag(question, language, tenant, chat_session) cache_manager.chat_session_cache.add_completed_interaction(session_id, new_interaction)
result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
result['interaction_id'] = interaction.id
result['room'] = room # Include the room in the result
if result['insufficient_info']: # Prepare response
if 'LLM' in tenant.fallback_algorithms: response = {
with current_event.create_span("Fallback Algorithm LLM"): 'result': result.model_dump(),
result, interaction = answer_using_llm(question, language, tenant, chat_session) 'interaction_id': new_interaction.id,
result['algorithm'] = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name'] 'room': room
result['interaction_id'] = interaction.id
result['room'] = room # Include the room in the result
return result
except Exception as e:
current_app.logger.error(f'ask_question: Error processing question: {e}')
raise
def answer_using_tenant_rag(question, language, tenant, chat_session):
new_interaction = Interaction()
new_interaction.question = question
new_interaction.language = language
new_interaction.timezone = chat_session.timezone
new_interaction.appreciation = None
new_interaction.chat_session_id = chat_session.id
new_interaction.question_at = dt.now(tz.utc)
new_interaction.algorithm_used = current_app.config['INTERACTION_ALGORITHMS']['RAG_TENANT']['name']
# Select variables to work with depending on tenant model
model_variables = select_model_variables(tenant)
tenant_info = tenant.to_dict()
# Langchain debugging if required
# set_debug(True)
with current_event.create_span("Detail Question"):
detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Detailed Question for tenant {tenant.id}:\n{question}.')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
new_interaction.detailed_question = detailed_question
new_interaction.detailed_question_at = dt.now(tz.utc)
with current_event.create_span("Generate Answer using RAG"):
retriever = EveAIDefaultRagRetriever(1, model_variables, tenant_info)
llm = model_variables['llm']
template = model_variables['rag_template']
language_template = create_language_template(template, language)
full_template = replace_variable_in_template(language_template, "{tenant_context}", model_variables['rag_context'])
rag_prompt = ChatPromptTemplate.from_template(full_template)
setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()})
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Full prompt for tenant {tenant.id}:\n{full_template}.')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
new_interaction_embeddings = []
if not model_variables['cited_answer_cls']: # The model doesn't support structured feedback
output_parser = StrOutputParser()
chain = setup_and_retrieval | rag_prompt | llm | output_parser
# Invoke the chain with the actual question
answer = chain.invoke(detailed_question)
new_interaction.answer = answer
result = {
'answer': answer,
'citations': [],
'insufficient_info': False
} }
else: # The model supports structured feedback return response
structured_llm = llm.with_structured_output(model_variables['cited_answer_cls'])
chain = setup_and_retrieval | rag_prompt | structured_llm except Exception as e:
current_app.logger.error(f'execute_specialist: Error executing specialist: {e}')
result = chain.invoke(detailed_question).dict()
current_app.logger.debug(f'ask_question: result answer: {result['answer']}')
current_app.logger.debug(f'ask_question: result citations: {result["citations"]}')
current_app.logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'ask_question: result answer: {result['answer']}')
current_app.rag_tuning_logger.debug(f'ask_question: result citations: {result["citations"]}')
current_app.rag_tuning_logger.debug(f'ask_question: insufficient information: {result["insufficient_info"]}')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
new_interaction.answer = result['answer']
# Filter out the existing Embedding IDs
given_embedding_ids = [int(emb_id) for emb_id in result['citations']]
embeddings = (
db.session.query(Embedding)
.filter(Embedding.id.in_(given_embedding_ids))
.all()
)
existing_embedding_ids = [emb.id for emb in embeddings]
urls = list(set(emb.document_version.url for emb in embeddings))
if model_variables['rag_tuning']:
current_app.rag_tuning_logger.debug(f'Referenced documents for answer for tenant {tenant.id}:\n')
current_app.rag_tuning_logger.debug(f'{urls}')
current_app.rag_tuning_logger.debug(f'-------------------------------------------------------------------')
for emb_id in existing_embedding_ids:
new_interaction_embedding = InteractionEmbedding(embedding_id=emb_id)
new_interaction_embedding.interaction = new_interaction
new_interaction_embeddings.append(new_interaction_embedding)
result['citations'] = urls
# Disable langchain debugging if set above.
# set_debug(False)
new_interaction.answer_at = dt.now(tz.utc)
chat_session.session_end = dt.now(tz.utc)
try:
db.session.add(chat_session)
db.session.add(new_interaction)
db.session.add_all(new_interaction_embeddings)
db.session.commit()
return result, new_interaction
except SQLAlchemyError as e:
current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
raise
def answer_using_llm(question, language, tenant, chat_session):
new_interaction = Interaction()
new_interaction.question = question
new_interaction.language = language
new_interaction.timezone = chat_session.timezone
new_interaction.appreciation = None
new_interaction.chat_session_id = chat_session.id
new_interaction.question_at = dt.now(tz.utc)
new_interaction.algorithm_used = current_app.config['INTERACTION_ALGORITHMS']['LLM']['name']
# Select variables to work with depending on tenant model
model_variables = select_model_variables(tenant)
tenant_info = tenant.to_dict()
# Langchain debugging if required
# set_debug(True)
with current_event.create_span("Detail Question"):
detailed_question = detail_question(question, language, model_variables, chat_session.session_id)
current_app.logger.debug(f'Original question:\n {question}\n\nDetailed question: {detailed_question}')
new_interaction.detailed_question = detailed_question
new_interaction.detailed_question_at = dt.now(tz.utc)
with current_event.create_span("Detail Answer using LLM"):
retriever = EveAIDefaultRagRetriever(1, model_variables, tenant_info)
llm = model_variables['llm_no_rag']
template = model_variables['encyclopedia_template']
language_template = create_language_template(template, language)
rag_prompt = ChatPromptTemplate.from_template(language_template)
setup = RunnablePassthrough()
output_parser = StrOutputParser()
new_interaction_embeddings = []
chain = setup | rag_prompt | llm | output_parser
input_question = {"question": detailed_question}
# Invoke the chain with the actual question
answer = chain.invoke(input_question)
new_interaction.answer = answer
result = {
'answer': answer,
'citations': [],
'insufficient_info': False
}
# Disable langchain debugging if set above.
# set_debug(False)
new_interaction.answer_at = dt.now(tz.utc)
chat_session.session_end = dt.now(tz.utc)
try:
db.session.add(chat_session)
db.session.add(new_interaction)
db.session.commit()
return result, new_interaction
except SQLAlchemyError as e:
current_app.logger.error(f'ask_question: Error saving interaction to database: {e}')
raise raise

9
eveai_entitlements/tasks.py
View File

@@ -45,20 +45,14 @@ def update_usages():
max_timestamp = max(log.timestamp for log in logs) max_timestamp = max(log.timestamp for log in logs)
# Retrieve relevant LicenseUsage records # Retrieve relevant LicenseUsage records
current_app.logger.debug(f"Searching relevant usages for tenant {tenant_id}")
license_usages = get_relevant_license_usages(db.session, tenant_id, min_timestamp, max_timestamp) license_usages = get_relevant_license_usages(db.session, tenant_id, min_timestamp, max_timestamp)
current_app.logger.debug(f"Found {license_usages}, end searching relevant usages for tenant {tenant_id}")
# Split logs based on LicenseUsage periods # Split logs based on LicenseUsage periods
current_app.logger.debug(f"Splitting usages for tenant {tenant_id}")
logs_by_usage = split_logs_by_license_usage(logs, license_usages) logs_by_usage = split_logs_by_license_usage(logs, license_usages)
current_app.logger.debug(f"Found {logs_by_usage}, end splitting logs for tenant {tenant_id}")
# Now you can process logs for each LicenseUsage # Now you can process logs for each LicenseUsage
for license_usage_id, logs in logs_by_usage.items(): for license_usage_id, logs in logs_by_usage.items():
current_app.logger.debug(f"Processing logs for usage id {license_usage_id} for tenant {tenant_id}")
process_logs_for_license_usage(tenant_id, license_usage_id, logs) process_logs_for_license_usage(tenant_id, license_usage_id, logs)
current_app.logger.debug(f"Finished processing logs for tenant {tenant_id}")
except Exception as e: except Exception as e:
error = f"Usage Calculation error for Tenant {tenant_id}: {e}" error = f"Usage Calculation error for Tenant {tenant_id}: {e}"
error_list.append(error) error_list.append(error)
@@ -212,8 +206,6 @@ def process_logs_for_license_usage(tenant_id, license_usage_id, logs):
license_usage.interaction_completion_tokens_used += interaction_completion_tokens_used license_usage.interaction_completion_tokens_used += interaction_completion_tokens_used
license_usage.interaction_total_tokens_used += interaction_total_tokens_used license_usage.interaction_total_tokens_used += interaction_total_tokens_used
current_app.logger.debug(f"Processed logs for license usage {license_usage.id}:\n{license_usage}")
# Commit the updates to the LicenseUsage and log records # Commit the updates to the LicenseUsage and log records
try: try:
db.session.add(license_usage) db.session.add(license_usage)
@@ -232,7 +224,6 @@ def recalculate_storage_for_tenant(tenant):
SELECT SUM(file_size) SELECT SUM(file_size)
FROM document_version FROM document_version
""")).scalar() """)).scalar()
current_app.logger.debug(f"Recalculating storage for tenant {tenant} - Total storage: {total_storage}")
# Update the LicenseUsage with the recalculated storage # Update the LicenseUsage with the recalculated storage
license_usage = db.session.query(LicenseUsage).filter_by(tenant_id=tenant.id).first() license_usage = db.session.query(LicenseUsage).filter_by(tenant_id=tenant.id).first()

11
eveai_workers/__init__.py
View File

@@ -4,10 +4,12 @@ from flask import Flask
import os import os
from common.utils.celery_utils import make_celery, init_celery from common.utils.celery_utils import make_celery, init_celery
from common.extensions import db, minio_client from common.extensions import db, minio_client, template_manager, cache_manager
from config.logging_config import LOGGING import config.logging_config as logging_config
from config.config import get_config from config.config import get_config
from . import processors
def create_app(config_file=None): def create_app(config_file=None):
app = Flask(__name__) app = Flask(__name__)
@@ -22,8 +24,7 @@ def create_app(config_file=None):
case _: case _:
app.config.from_object(get_config('dev')) app.config.from_object(get_config('dev'))
logging.config.dictConfig(LOGGING) logging.config.dictConfig(logging_config.LOGGING)
app.embed_tuning_logger = logging.getLogger('embed_tuning')
register_extensions(app) register_extensions(app)
@@ -41,6 +42,8 @@ def create_app(config_file=None):
def register_extensions(app): def register_extensions(app):
db.init_app(app) db.init_app(app)
minio_client.init_app(app) minio_client.init_app(app)
cache_manager.init_app(app)
template_manager.init_app(app)
app, celery = create_app() app, celery = create_app()

5
eveai_workers/processors/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
# Import all processor implementations to ensure registration
from . import audio_processor, html_processor, pdf_processor
# List of all available processor implementations
__all__ = ['audio_processor', 'html_processor', 'pdf_processor']

75
eveai_workers/Processors/audio_processor.py → eveai_workers/processors/audio_processor.py
View File

@@ -8,20 +8,20 @@ import tempfile
from common.extensions import minio_client from common.extensions import minio_client
import subprocess import subprocess
from .transcription_processor import TranscriptionProcessor from .processor_registry import ProcessorRegistry
from .transcription_processor import TranscriptionBaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
class AudioProcessor(TranscriptionProcessor): class AudioProcessor(TranscriptionBaseProcessor):
def __init__(self, tenant, model_variables, document_version): def __init__(self, tenant, model_variables, document_version, catalog, processor):
super().__init__(tenant, model_variables, document_version) super().__init__(tenant, model_variables, document_version, catalog, processor)
self.transcription_client = model_variables['transcription_client'] self.transcription_model = model_variables.transcription_model
self.transcription_model = model_variables['transcription_model']
self.ffmpeg_path = 'ffmpeg' self.ffmpeg_path = 'ffmpeg'
self.max_compression_duration = model_variables['max_compression_duration'] self.max_compression_duration = model_variables.max_compression_duration
self.max_transcription_duration = model_variables['max_transcription_duration'] self.max_transcription_duration = model_variables.max_transcription_duration
self.compression_cpu_limit = model_variables.get('compression_cpu_limit', 50) # CPU usage limit in percentage self.compression_cpu_limit = model_variables.compression_cpu_limit # CPU usage limit in percentage
self.compression_process_delay = model_variables.get('compression_process_delay', 0.1) # Delay between processing chunks in seconds self.compression_process_delay = model_variables.compression_process_delay # Delay between processing chunks in seconds
self.file_type = document_version.file_type self.file_type = document_version.file_type
def _get_transcription(self): def _get_transcription(self):
@@ -39,26 +39,25 @@ class AudioProcessor(TranscriptionProcessor):
return transcription return transcription
def _compress_audio(self, audio_data): def _compress_audio(self, audio_data):
self._log("Compressing audio")
with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{self.document_version.file_type}') as temp_file: with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{self.document_version.file_type}') as temp_file:
temp_file.write(audio_data) temp_file.write(audio_data)
temp_file_path = temp_file.name temp_file_path = temp_file.name
try: try:
self._log("Creating AudioSegment from file")
audio_info = AudioSegment.from_file(temp_file_path, format=self.document_version.file_type) audio_info = AudioSegment.from_file(temp_file_path, format=self.document_version.file_type)
self._log("Finished creating AudioSegment from file")
total_duration = len(audio_info) total_duration = len(audio_info)
self._log(f"Audio duration: {total_duration / 1000} seconds") self._log_tuning("_compress_audio", {
"Audio Duration (ms)": total_duration,
})
segment_length = self.max_compression_duration * 1000 # Convert to milliseconds segment_length = self.max_compression_duration * 1000 # Convert to milliseconds
total_chunks = (total_duration + segment_length - 1) // segment_length total_chunks = (total_duration + segment_length - 1) // segment_length
compressed_segments = AudioSegment.empty() compressed_segments = AudioSegment.empty()
for i in range(total_chunks): for i in range(total_chunks):
self._log(f"Compressing segment {i + 1} of {total_chunks}") self._log_tuning("_compress_audio", {
"Segment Nr": f"{i + 1} of {total_chunks}"
})
start_time = i * segment_length start_time = i * segment_length
end_time = min((i + 1) * segment_length, total_duration) end_time = min((i + 1) * segment_length, total_duration)
@@ -88,7 +87,9 @@ class AudioProcessor(TranscriptionProcessor):
compressed_filename, compressed_filename,
compressed_buffer.read() compressed_buffer.read()
) )
self._log(f"Saved compressed audio to MinIO: {compressed_filename}") self._log_tuning("_compress_audio", {
"Compressed audio to MinIO": compressed_filename
})
return compressed_segments return compressed_segments
@@ -131,7 +132,6 @@ class AudioProcessor(TranscriptionProcessor):
return compressed_segment return compressed_segment
def _transcribe_audio(self, audio_data): def _transcribe_audio(self, audio_data):
self._log("Starting audio transcription")
# audio = AudioSegment.from_file(io.BytesIO(audio_data), format="mp3") # audio = AudioSegment.from_file(io.BytesIO(audio_data), format="mp3")
audio = audio_data audio = audio_data
@@ -140,7 +140,6 @@ class AudioProcessor(TranscriptionProcessor):
total_chunks = len(audio) // segment_length + 1 total_chunks = len(audio) // segment_length + 1
for i, chunk in enumerate(audio[::segment_length]): for i, chunk in enumerate(audio[::segment_length]):
self._log(f'Processing chunk {i + 1} of {total_chunks}')
segment_duration = 0 segment_duration = 0
if i == total_chunks - 1: if i == total_chunks - 1:
segment_duration = (len(audio) % segment_length) // 1000 segment_duration = (len(audio) % segment_length) // 1000
@@ -153,37 +152,34 @@ class AudioProcessor(TranscriptionProcessor):
try: try:
file_size = os.path.getsize(temp_audio.name) file_size = os.path.getsize(temp_audio.name)
self._log(f"Temporary audio file size: {file_size} bytes")
with open(temp_audio.name, 'rb') as audio_file: with open(temp_audio.name, 'rb') as audio_file:
file_start = audio_file.read(100) transcription = self.model_variables.transcription_model.transcribe(
self._log(f"First 100 bytes of audio file: {file_start}")
audio_file.seek(0) # Reset file pointer to the beginning
self._log("Calling transcription API")
transcription = self.model_variables.transcribe(
file=audio_file, file=audio_file,
model=self.transcription_model,
language=self.document_version.language, language=self.document_version.language,
response_format='verbose_json', response_format='verbose_json',
duration=segment_duration, duration=segment_duration
) )
self._log("Transcription API call completed")
if transcription: if transcription:
trans = ""
# Handle the transcription result based on its type # Handle the transcription result based on its type
if isinstance(transcription, str): if isinstance(transcription, str):
self._log(f"Transcription result (string): {transcription[:100]}...") trans = transcription
transcriptions.append(transcription)
elif hasattr(transcription, 'text'): elif hasattr(transcription, 'text'):
self._log( trans = transcription.text
f"Transcription result (object with 'text' attribute): {transcription.text[:100]}...")
transcriptions.append(transcription.text)
else: else:
self._log(f"Transcription result (unknown type): {str(transcription)[:100]}...")
transcriptions.append(str(transcription)) transcriptions.append(str(transcription))
transcriptions.append(trans)
self._log_tuning("_transcribe_audio", {
"Chunk Nr": f"{i + 1} of {total_chunks}",
"Segment Duration": segment_duration,
"Transcription": trans,
})
else: else:
self._log("Warning: Received empty transcription", level='warning') self._log("Warning: Received empty transcription", level='warning')
self._log_tuning("_transcribe_audio", {"ERROR": "No transcription"})
except Exception as e: except Exception as e:
self._log(f"Error during transcription: {str(e)}", level='error') self._log(f"Error during transcription: {str(e)}", level='error')
@@ -206,7 +202,10 @@ class AudioProcessor(TranscriptionProcessor):
transcription_filename, transcription_filename,
full_transcription.encode('utf-8') full_transcription.encode('utf-8')
) )
self._log(f"Saved transcription to MinIO: {transcription_filename}") self._log_tuning(f"Saved transcription to MinIO: {transcription_filename}")
return full_transcription return full_transcription
# Register the processor
ProcessorRegistry.register("AUDIO_PROCESSOR", AudioProcessor)

42
eveai_workers/Processors/processor.py → eveai_workers/processors/base_processor.py
View File

@@ -1,14 +1,42 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Dict, Any
from flask import current_app from flask import current_app
from common.extensions import minio_client from common.extensions import minio_client
from config.logging_config import TuningLogger
class Processor(ABC): class BaseProcessor(ABC):
def __init__(self, tenant, model_variables, document_version): def __init__(self, tenant, model_variables, document_version, catalog, processor):
self.tenant = tenant self.tenant = tenant
self.model_variables = model_variables self.model_variables = model_variables
self.document_version = document_version self.document_version = document_version
self.embed_tuning = model_variables['embed_tuning'] self.catalog = catalog
self.processor = processor
self.tuning = processor.tuning if processor else False
self.tuning_logger = None
self._setup_tuning_logger()
self._log_tuning("Processor initialized", {
"processor_type": processor.type if processor else None,
"document_version": document_version.id if document_version else None,
"catalog": catalog.id if catalog else None
})
def _setup_tuning_logger(self):
try:
self.tuning_logger = TuningLogger(
'tuning',
tenant_id=self.tenant.id if self.tenant else None,
catalog_id=self.catalog.id if self.catalog else None,
processor_id=self.processor.id if self.processor else None,
)
# Verify logger is working with a test message
if self.tuning:
self.tuning_logger.log_tuning('processor', "Tuning logger initialized")
except Exception as e:
current_app.logger.error(f"Failed to setup tuning logger: {str(e)}")
raise
@abstractmethod @abstractmethod
def process(self): def process(self):
@@ -50,3 +78,11 @@ class Processor(ABC):
return markdown return markdown
def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None:
if self.tuning and self.tuning_logger:
try:
self.tuning_logger.log_tuning('processor', message, data)
except Exception as e:
current_app.logger.error(f"Processor: Error in tuning logging: {e}")

51
eveai_workers/Processors/html_processor.py → eveai_workers/processors/html_processor.py
View File

@@ -4,21 +4,34 @@ from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough from langchain_core.runnables import RunnablePassthrough
from common.extensions import db, minio_client from common.extensions import db, minio_client
from common.utils.model_utils import create_language_template from common.utils.model_utils import create_language_template
from .processor import Processor from .base_processor import BaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
from .processor_registry import ProcessorRegistry
from common.utils.string_list_converter import StringListConverter as SLC
class HTMLProcessor(Processor): class HTMLProcessor(BaseProcessor):
def __init__(self, tenant, model_variables, document_version): def __init__(self, tenant, model_variables, document_version, catalog, processor):
super().__init__(tenant, model_variables, document_version) super().__init__(tenant, model_variables, document_version, catalog, processor)
self.html_tags = model_variables['html_tags'] cat_conf = catalog.configuration
self.html_end_tags = model_variables['html_end_tags'] proc_conf = processor.configuration
self.html_included_elements = model_variables['html_included_elements'] self.html_tags = SLC.string_to_list(proc_conf['html_tags'])
self.html_excluded_elements = model_variables['html_excluded_elements'] self.html_end_tags = SLC.string_to_list(proc_conf['html_end_tags'])
self.html_excluded_classes = model_variables['html_excluded_classes'] self.html_included_elements = SLC.string_to_list(proc_conf['html_included_elements'])
self.chunk_size = model_variables['processing_chunk_size'] # Adjust this based on your LLM's optimal input size self.html_excluded_elements = SLC.string_to_list(proc_conf['html_excluded_elements'])
self.chunk_overlap = model_variables[ self.html_excluded_classes = SLC.string_to_list(proc_conf['html_excluded_classes'])
'processing_chunk_overlap'] # Adjust for context preservation between chunks self.tuning = self.processor.tuning
# Add verification logging
self._log(f"HTML Processor initialized with tuning={self.tuning}")
if self.tuning:
self._log_tuning("HTML Processor initialized", {
"html_tags": self.html_tags,
"html_end_tags": self.html_end_tags,
"included_elements": self.html_included_elements,
"excluded_elements": self.html_excluded_elements
})
self.chunk_size = catalog.max_chunk_size
def process(self): def process(self):
self._log("Starting HTML processing") self._log("Starting HTML processing")
@@ -62,13 +75,14 @@ class HTMLProcessor(Processor):
title = soup.find('title').get_text(strip=True) if soup.find('title') else '' title = soup.find('title').get_text(strip=True) if soup.find('title') else ''
self._log(f'Finished parsing HTML for tenant {self.tenant.id}') self._log(f'Finished parsing HTML for tenant {self.tenant.id}')
self._log_tuning("_parse_html", {"extracted_html": extracted_html, "title": title})
return extracted_html, title return extracted_html, title
def _generate_markdown_from_html(self, html_content): def _generate_markdown_from_html(self, html_content):
self._log(f'Generating markdown from HTML for tenant {self.tenant.id}') self._log(f'Generating markdown from HTML for tenant {self.tenant.id}')
llm = self.model_variables['llm'] llm = self.model_variables.get_llm()
template = self.model_variables['html_parse_template'] template = self.model_variables.get_template("html_parse")
parse_prompt = ChatPromptTemplate.from_template(template) parse_prompt = ChatPromptTemplate.from_template(template)
setup = RunnablePassthrough() setup = RunnablePassthrough()
output_parser = StrOutputParser() output_parser = StrOutputParser()
@@ -79,13 +93,10 @@ class HTMLProcessor(Processor):
markdown_chunks = [] markdown_chunks = []
for chunk in chunks: for chunk in chunks:
if self.embed_tuning:
self._log(f'Processing chunk: \n{chunk}\n')
input_html = {"html": chunk} input_html = {"html": chunk}
markdown_chunk = chain.invoke(input_html) markdown_chunk = chain.invoke(input_html)
markdown_chunks.append(markdown_chunk) markdown_chunks.append(markdown_chunk)
if self.embed_tuning: self._log_tuning("_generate_markdown_from_html", {"chunk": chunk, "markdown_chunk": markdown_chunk})
self._log(f'Processed markdown chunk: \n{markdown_chunk}\n')
markdown = "\n\n".join(markdown_chunks) markdown = "\n\n".join(markdown_chunks)
self._log(f'Finished generating markdown from HTML for tenant {self.tenant.id}') self._log(f'Finished generating markdown from HTML for tenant {self.tenant.id}')
@@ -146,3 +157,7 @@ class HTMLProcessor(Processor):
def _extract_element_content(self, element): def _extract_element_content(self, element):
content = ' '.join(child.strip() for child in element.stripped_strings) content = ' '.join(child.strip() for child in element.stripped_strings)
return f'<{element.name}>{content}</{element.name}>\n' return f'<{element.name}>{content}</{element.name}>\n'
# Register the processor
ProcessorRegistry.register("HTML_PROCESSOR", HTMLProcessor)

43
eveai_workers/Processors/pdf_processor.py → eveai_workers/processors/pdf_processor.py
View File

@@ -9,18 +9,18 @@ from langchain_core.runnables import RunnablePassthrough
from common.extensions import minio_client from common.extensions import minio_client
from common.utils.model_utils import create_language_template from common.utils.model_utils import create_language_template
from .processor import Processor from .base_processor import BaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
from .processor_registry import ProcessorRegistry
class PDFProcessor(Processor): class PDFProcessor(BaseProcessor):
def __init__(self, tenant, model_variables, document_version): def __init__(self, tenant, model_variables, document_version, catalog, processor):
super().__init__(tenant, model_variables, document_version) super().__init__(tenant, model_variables, document_version, catalog, processor)
# PDF-specific initialization
self.chunk_size = model_variables['processing_chunk_size'] self.chunk_size = catalog.max_chunk_size
self.chunk_overlap = model_variables['processing_chunk_overlap'] self.chunk_overlap = 0
self.min_chunk_size = model_variables['processing_min_chunk_size'] self.tuning = self.processor.tuning
self.max_chunk_size = model_variables['processing_max_chunk_size']
def process(self): def process(self):
self._log("Starting PDF processing") self._log("Starting PDF processing")
@@ -38,7 +38,8 @@ class PDFProcessor(Processor):
with current_event.create_span("Markdown Generation"): with current_event.create_span("Markdown Generation"):
llm_chunks = self._split_content_for_llm(structured_content) llm_chunks = self._split_content_for_llm(structured_content)
markdown = self._process_chunks_with_llm(llm_chunks) markdown = self._process_chunks_with_llm(llm_chunks)
self._save_markdown(markdown)
self._save_markdown(markdown)
self._log("Finished processing PDF") self._log("Finished processing PDF")
return markdown, title return markdown, title
except Exception as e: except Exception as e:
@@ -56,19 +57,10 @@ class PDFProcessor(Processor):
'figures': self._extract_figures(page, page_num, figure_counter), 'figures': self._extract_figures(page, page_num, figure_counter),
'tables': self._extract_tables(page) 'tables': self._extract_tables(page)
} }
if self.embed_tuning: self._log_tuning("_extract_content", {"page_num": page_num, "page_content": page_content})
self._log(f'Extracted PDF Content for page {page_num + 1}')
self._log(f"{page_content }")
figure_counter += len(page_content['figures']) figure_counter += len(page_content['figures'])
extracted_content.append(page_content) extracted_content.append(page_content)
# if self.embed_tuning:
# current_app.embed_tuning_logger.debug(f'Extracted PDF Content')
# current_app.embed_tuning_logger.debug(f'---------------------')
# current_app.embed_tuning_logger.debug(f'Page: {page_content}')
# current_app.embed_tuning_logger.debug(f'End of Extracted PDF Content')
# current_app.embed_tuning_logger.debug(f'----------------------------')
return extracted_content return extracted_content
def _extract_figures(self, page, page_num, figure_counter): def _extract_figures(self, page, page_num, figure_counter):
@@ -127,6 +119,7 @@ class PDFProcessor(Processor):
markdown_table = self._table_to_markdown(table) markdown_table = self._table_to_markdown(table)
if markdown_table: # Only add non-empty tables if markdown_table: # Only add non-empty tables
tables.append(markdown_table) tables.append(markdown_table)
self._log_tuning("_extract_tables", {"markdown_table": markdown_table})
except Exception as e: except Exception as e:
self._log(f"Error extracting tables from page: {str(e)}", level='error') self._log(f"Error extracting tables from page: {str(e)}", level='error')
return tables return tables
@@ -202,7 +195,7 @@ class PDFProcessor(Processor):
for table in page['tables']: for table in page['tables']:
structured_content += f"\n{table}\n" structured_content += f"\n{table}\n"
if self.embed_tuning: if self.tuning:
self._save_intermediate(structured_content, "structured_content.md") self._save_intermediate(structured_content, "structured_content.md")
return structured_content, title return structured_content, title
@@ -217,8 +210,8 @@ class PDFProcessor(Processor):
return text_splitter.split_text(content) return text_splitter.split_text(content)
def _process_chunks_with_llm(self, chunks): def _process_chunks_with_llm(self, chunks):
llm = self.model_variables['llm'] llm = self.model_variables.get_llm()
template = self.model_variables['pdf_parse_template'] template = self.model_variables.get_template('pdf_parse')
pdf_prompt = ChatPromptTemplate.from_template(template) pdf_prompt = ChatPromptTemplate.from_template(template)
setup = RunnablePassthrough() setup = RunnablePassthrough()
output_parser = StrOutputParser() output_parser = StrOutputParser()
@@ -232,3 +225,7 @@ class PDFProcessor(Processor):
markdown_chunks.append(result) markdown_chunks.append(result)
return "\n\n".join(markdown_chunks) return "\n\n".join(markdown_chunks)
# Register the processor
ProcessorRegistry.register("PDF_PROCESSOR", PDFProcessor)

92
eveai_workers/processors/processor_registry.py Normal file
View File

@@ -0,0 +1,92 @@
from typing import Dict, Type, Optional
from flask import current_app
from config.processor_types import PROCESSOR_TYPES
from .base_processor import BaseProcessor
class ProcessorRegistry:
"""Registry for processor types that aligns with PROCESSOR_TYPES configuration"""
_registry: Dict[str, Type[BaseProcessor]] = {}
@classmethod
def register(cls, processor_type: str, processor_class: Type[BaseProcessor]):
"""
Register a new processor type that must match a type in PROCESSOR_TYPES
Args:
processor_type: Type identifier from PROCESSOR_TYPES
processor_class: Processor implementation class
Raises:
ValueError: If processor_type isn't defined in PROCESSOR_TYPES
"""
if processor_type not in PROCESSOR_TYPES:
raise ValueError(f"Processor type {processor_type} not found in PROCESSOR_TYPES configuration")
cls._registry[processor_type] = processor_class
@classmethod
def get_processor_class(cls, processor_type: str) -> Type[BaseProcessor]:
"""
Get the processor class for a given processor type
Args:
processor_type: Type identifier from PROCESSOR_TYPES
Returns:
The registered processor class
Raises:
ValueError: If no processor is registered for the given type
"""
if processor_type not in cls._registry:
raise ValueError(f"No processor registered for type: {processor_type}")
return cls._registry[processor_type]
@classmethod
def get_processor_for_file_type(cls, file_type: str) -> tuple[str, Type[BaseProcessor]]:
"""
Find appropriate processor for a file type by checking PROCESSOR_TYPES definitions
Args:
file_type: File extension (e.g., 'html', 'pdf')
Returns:
Tuple of (processor_type, processor_class)
Raises:
ValueError: If no processor is found for the file type
"""
# First find which processor type handles this file type
for proc_type, config in PROCESSOR_TYPES.items():
# Check if file_type is in the supported file_types (handling both string and list formats)
supported_types = config['file_types']
if isinstance(supported_types, str):
supported_types = [t.strip() for t in supported_types.split(',')]
if file_type in supported_types:
# Get the registered processor class for this type
if proc_type in cls._registry:
return proc_type, cls._registry[proc_type]
else:
raise ValueError(
f"Found processor type {proc_type} for file type {file_type} but no processor is registered")
raise ValueError(f"No processor type found for file type: {file_type}")
@classmethod
def validate_processor_registration(cls):
"""
Validate that all PROCESSOR_TYPES have registered processors
Raises:
ValueError: If any processor type lacks a registered processor
"""
missing_processors = []
for proc_type in PROCESSOR_TYPES.keys():
if proc_type not in cls._registry:
missing_processors.append(proc_type)
if missing_processors:
raise ValueError(f"Missing processor registrations for: {', '.join(missing_processors)}")

4
eveai_workers/Processors/srt_processor.py → eveai_workers/processors/srt_processor.py
View File

@@ -1,9 +1,9 @@
from common.extensions import minio_client from common.extensions import minio_client
from .transcription_processor import TranscriptionProcessor from .transcription_processor import TranscriptionBaseProcessor
import re import re
class SRTProcessor(TranscriptionProcessor): class SRTProcessor(TranscriptionBaseProcessor):
def _get_transcription(self): def _get_transcription(self):
file_data = minio_client.download_document_file( file_data = minio_client.download_document_file(
self.tenant.id, self.tenant.id,

30
eveai_workers/Processors/transcription_processor.py → eveai_workers/processors/transcription_processor.py
View File

@@ -5,15 +5,15 @@ from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough from langchain_core.runnables import RunnablePassthrough
from common.utils.model_utils import create_language_template from common.utils.model_utils import create_language_template
from .processor import Processor from .base_processor import BaseProcessor
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
class TranscriptionProcessor(Processor): class TranscriptionBaseProcessor(BaseProcessor):
def __init__(self, tenant, model_variables, document_version): def __init__(self, tenant, model_variables, document_version, catalog, processor):
super().__init__(tenant, model_variables, document_version) super().__init__(tenant, model_variables, document_version, catalog, processor)
self.chunk_size = model_variables['processing_chunk_size'] self.annotation_chunk_size = model_variables.annotation_chunk_length
self.chunk_overlap = model_variables['processing_chunk_overlap'] self.annotation_chunk_overlap = 0
def process(self): def process(self):
self._log("Starting Transcription processing") self._log("Starting Transcription processing")
@@ -37,17 +37,17 @@ class TranscriptionProcessor(Processor):
def _chunk_transcription(self, transcription): def _chunk_transcription(self, transcription):
text_splitter = RecursiveCharacterTextSplitter( text_splitter = RecursiveCharacterTextSplitter(
chunk_size=self.chunk_size, chunk_size=self.annotation_chunk_size,
chunk_overlap=self.chunk_overlap, chunk_overlap=self.annotation_chunk_overlap,
length_function=len, length_function=len,
separators=["\n\n", "\n", " ", ""] separators=["\n\n", "\n", " ", ""]
) )
return text_splitter.split_text(transcription) return text_splitter.split_text(transcription)
def _process_chunks(self, chunks): def _process_chunks(self, chunks):
self._log("Generating markdown from transcription") self._log_tuning("_process_chunks", {"Nr of Chunks": len(chunks)})
llm = self.model_variables['llm'] llm = self.model_variables.get_llm()
template = self.model_variables['transcript_template'] template = self.model_variables.get_template('transcript')
language_template = create_language_template(template, self.document_version.language) language_template = create_language_template(template, self.document_version.language)
transcript_prompt = ChatPromptTemplate.from_template(language_template) transcript_prompt = ChatPromptTemplate.from_template(language_template)
setup = RunnablePassthrough() setup = RunnablePassthrough()
@@ -58,14 +58,18 @@ class TranscriptionProcessor(Processor):
markdown_chunks = [] markdown_chunks = []
previous_part = "" previous_part = ""
for i, chunk in enumerate(chunks): for i, chunk in enumerate(chunks):
self._log(f"Processing chunk {i + 1} of {len(chunks)}")
self._log(f"Previous part: {previous_part}")
input_transcript = { input_transcript = {
'transcript': chunk, 'transcript': chunk,
'previous_part': previous_part 'previous_part': previous_part
} }
markdown = chain.invoke(input_transcript) markdown = chain.invoke(input_transcript)
markdown = self._clean_markdown(markdown) markdown = self._clean_markdown(markdown)
self._log_tuning("_process_chunks", {
"Chunk Number": f"{i + 1} of {len(chunks)}",
"Chunk": chunk,
"Previous Chunk": previous_part,
"Markdown": markdown,
})
markdown_chunks.append(markdown) markdown_chunks.append(markdown)
# Extract the last part for the next iteration # Extract the last part for the next iteration

198
eveai_workers/tasks.py
View File

@@ -10,22 +10,20 @@ from langchain_core.exceptions import LangChainException
from langchain_core.output_parsers import StrOutputParser from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough from langchain_core.runnables import RunnablePassthrough
from sqlalchemy import or_
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from common.extensions import db, minio_client from common.extensions import db, minio_client, template_manager
from common.models.document import DocumentVersion, Embedding, Document from common.models.document import DocumentVersion, Embedding, Document, Processor, Catalog
from common.models.user import Tenant from common.models.user import Tenant
from common.utils.celery_utils import current_celery from common.utils.celery_utils import current_celery
from common.utils.database import Database from common.utils.database import Database
from common.utils.model_utils import select_model_variables, create_language_template from common.utils.model_utils import create_language_template, get_model_variables
from common.utils.os_utils import safe_remove, sync_folder
from eveai_workers.Processors.audio_processor import AudioProcessor
from eveai_workers.Processors.html_processor import HTMLProcessor
from eveai_workers.Processors.pdf_processor import PDFProcessor
from eveai_workers.Processors.srt_processor import SRTProcessor
from common.utils.business_event import BusinessEvent from common.utils.business_event import BusinessEvent
from common.utils.business_event_context import current_event from common.utils.business_event_context import current_event
from config.processor_types import PROCESSOR_TYPES
from eveai_workers.processors.processor_registry import ProcessorRegistry
# Healthcheck task # Healthcheck task
@@ -53,14 +51,18 @@ def create_embeddings(tenant_id, document_version_id):
# Retrieve the Catalog ID # Retrieve the Catalog ID
doc = Document.query.get_or_404(document_version.doc_id) doc = Document.query.get_or_404(document_version.doc_id)
catalog_id = doc.catalog_id catalog_id = doc.catalog_id
catalog = Catalog.query.get_or_404(catalog_id)
# Select variables to work with depending on tenant and model # Select variables to work with depending on tenant and model
model_variables = select_model_variables(tenant, catalog_id=catalog_id) model_variables = get_model_variables(tenant_id)
current_app.logger.debug(f'Model variables: {model_variables}')
# Define processor related information
processor_type, processor_class = ProcessorRegistry.get_processor_for_file_type(document_version.file_type)
processor = get_processor_for_document(catalog_id, document_version.file_type, document_version.sub_file_type)
except Exception as e: except Exception as e:
current_app.logger.error(f'Create Embeddings request received ' current_app.logger.error(f'Create Embeddings request received '
f'for non existing document version {document_version_id} ' f'for badly configured document version {document_version_id} '
f'for tenant {tenant_id}, ' f'for tenant {tenant_id}, '
f'error: {e}') f'error: {e}')
raise raise
@@ -90,19 +92,19 @@ def create_embeddings(tenant_id, document_version_id):
delete_embeddings_for_document_version(document_version) delete_embeddings_for_document_version(document_version)
try: try:
match document_version.file_type: with current_event.create_span(f"{processor_type} Processing"):
case 'pdf': document_processor = processor_class(
process_pdf(tenant, model_variables, document_version) tenant=tenant,
case 'html': model_variables=model_variables,
process_html(tenant, model_variables, document_version) document_version=document_version,
case 'srt': catalog=catalog,
process_srt(tenant, model_variables, document_version) processor=processor
case 'mp4' | 'mp3' | 'ogg': )
process_audio(tenant, model_variables, document_version) markdown, title = document_processor.process()
case _:
raise Exception(f'No functionality defined for file type {document_version.file_type} ' with current_event.create_span("Embedding"):
f'for tenant {tenant_id} ' embed_markdown(tenant, model_variables, document_version, catalog, markdown, title)
f'while creating embeddings for document version {document_version_id}')
current_event.log("Finished Embedding Creation Task") current_event.log("Finished Embedding Creation Task")
except Exception as e: except Exception as e:
@@ -129,53 +131,12 @@ def delete_embeddings_for_document_version(document_version):
raise raise
def process_pdf(tenant, model_variables, document_version): def embed_markdown(tenant, model_variables, document_version, catalog, markdown, title):
with current_event.create_span("PDF Processing"):
processor = PDFProcessor(tenant, model_variables, document_version)
markdown, title = processor.process()
# Process markdown and embed
with current_event.create_span("Embedding"):
embed_markdown(tenant, model_variables, document_version, markdown, title)
def process_html(tenant, model_variables, document_version):
with current_event.create_span("HTML Processing"):
processor = HTMLProcessor(tenant, model_variables, document_version)
markdown, title = processor.process()
# Process markdown and embed
with current_event.create_span("Embedding"):
embed_markdown(tenant, model_variables, document_version, markdown, title)
def process_audio(tenant, model_variables, document_version):
with current_event.create_span("Audio Processing"):
processor = AudioProcessor(tenant, model_variables, document_version)
markdown, title = processor.process()
# Process markdown and embed
with current_event.create_span("Embedding"):
embed_markdown(tenant, model_variables, document_version, markdown, title)
def process_srt(tenant, model_variables, document_version):
with current_event.create_span("SRT Processing"):
processor = SRTProcessor(tenant, model_variables, document_version)
markdown, title = processor.process()
# Process markdown and embed
with current_event.create_span("Embedding"):
embed_markdown(tenant, model_variables, document_version, markdown, title)
def embed_markdown(tenant, model_variables, document_version, markdown, title):
# Create potential chunks # Create potential chunks
potential_chunks = create_potential_chunks_for_markdown(tenant.id, document_version, f"{document_version.id}.md") potential_chunks = create_potential_chunks_for_markdown(tenant.id, document_version, f"{document_version.id}.md")
# Combine chunks for embedding # Combine chunks for embedding
chunks = combine_chunks_for_markdown(potential_chunks, model_variables['min_chunk_size'], chunks = combine_chunks_for_markdown(potential_chunks, catalog.min_chunk_size, catalog.max_chunk_size)
model_variables['max_chunk_size'])
# Enrich chunks # Enrich chunks
with current_event.create_span("Enrich Chunks"): with current_event.create_span("Enrich Chunks"):
@@ -203,9 +164,6 @@ def embed_markdown(tenant, model_variables, document_version, markdown, title):
def enrich_chunks(tenant, model_variables, document_version, title, chunks): def enrich_chunks(tenant, model_variables, document_version, title, chunks):
current_app.logger.debug(f'Enriching chunks for tenant {tenant.id} '
f'on document version {document_version.id}')
summary = '' summary = ''
if len(chunks) > 1: if len(chunks) > 1:
summary = summarize_chunk(tenant, model_variables, document_version, chunks[0]) summary = summarize_chunk(tenant, model_variables, document_version, chunks[0])
@@ -233,18 +191,13 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks):
enriched_chunk = f'{chunk_total_context}\n{chunk}' enriched_chunk = f'{chunk_total_context}\n{chunk}'
enriched_chunks.append(enriched_chunk) enriched_chunks.append(enriched_chunk)
current_app.logger.debug(f'Finished enriching chunks for tenant {tenant.id} '
f'on document version {document_version.id}')
return enriched_chunks return enriched_chunks
def summarize_chunk(tenant, model_variables, document_version, chunk): def summarize_chunk(tenant, model_variables, document_version, chunk):
current_event.log("Starting Summarizing Chunk") current_event.log("Starting Summarizing Chunk")
current_app.logger.debug(f'Summarizing chunk for tenant {tenant.id} ' llm = model_variables.get_llm()
f'on document version {document_version.id}') template = model_variables.get_template("summary")
llm = model_variables['llm']
template = model_variables['summary_template']
language_template = create_language_template(template, document_version.language) language_template = create_language_template(template, document_version.language)
summary_prompt = ChatPromptTemplate.from_template(language_template) summary_prompt = ChatPromptTemplate.from_template(language_template)
setup = RunnablePassthrough() setup = RunnablePassthrough()
@@ -253,11 +206,7 @@ def summarize_chunk(tenant, model_variables, document_version, chunk):
chain = setup | summary_prompt | llm | output_parser chain = setup | summary_prompt | llm | output_parser
try: try:
current_app.logger.debug(f'Starting summarizing chunk for tenant {tenant.id} '
f'on document version {document_version.id}')
summary = chain.invoke({"text": chunk}) summary = chain.invoke({"text": chunk})
current_app.logger.debug(f'Finished summarizing chunk for tenant {tenant.id} '
f'on document version {document_version.id}.')
current_event.log("Finished Summarizing Chunk") current_event.log("Finished Summarizing Chunk")
return summary return summary
except LangChainException as e: except LangChainException as e:
@@ -268,14 +217,10 @@ def summarize_chunk(tenant, model_variables, document_version, chunk):
def embed_chunks(tenant, model_variables, document_version, chunks): def embed_chunks(tenant, model_variables, document_version, chunks):
current_app.logger.debug(f'Embedding chunks for tenant {tenant.id} ' embedding_model = model_variables.embedding_model
f'on document version {document_version.id}')
embedding_model = model_variables['embedding_model']
try: try:
embeddings = embedding_model.embed_documents(chunks) embeddings = embedding_model.embed_documents(chunks)
current_app.logger.debug(f'Finished embedding chunks for tenant {tenant.id} '
f'on document version {document_version.id}')
except LangChainException as e: except LangChainException as e:
current_app.logger.error(f'Error creating embeddings for tenant {tenant.id} ' current_app.logger.error(f'Error creating embeddings for tenant {tenant.id} '
f'on document version {document_version.id} while calling OpenAI API' f'on document version {document_version.id} while calling OpenAI API'
@@ -285,28 +230,16 @@ def embed_chunks(tenant, model_variables, document_version, chunks):
# Add embeddings to the database # Add embeddings to the database
new_embeddings = [] new_embeddings = []
for chunk, embedding in zip(chunks, embeddings): for chunk, embedding in zip(chunks, embeddings):
new_embedding = model_variables['embedding_db_model']() new_embedding = model_variables.embedding_model_class()
new_embedding.document_version = document_version new_embedding.document_version = document_version
new_embedding.active = True new_embedding.active = True
new_embedding.chunk = chunk new_embedding.chunk = chunk
new_embedding.embedding = embedding new_embedding.embedding = embedding
new_embeddings.append(new_embedding) new_embeddings.append(new_embedding)
current_app.logger.debug(f'Finished embedding chunks for tenant {tenant.id} ')
return new_embeddings return new_embeddings
def log_parsing_info(tenant, tags, included_elements, excluded_elements, excluded_classes, elements_to_parse):
if tenant.embed_tuning:
current_app.embed_tuning_logger.debug(f'Tags to parse: {tags}')
current_app.embed_tuning_logger.debug(f'Included Elements: {included_elements}')
current_app.embed_tuning_logger.debug(f'Excluded Elements: {excluded_elements}')
current_app.embed_tuning_logger.debug(f'Excluded Classes: {excluded_classes}')
current_app.embed_tuning_logger.debug(f'Found {len(elements_to_parse)} elements to parse')
current_app.embed_tuning_logger.debug(f'First element to parse: {elements_to_parse[0]}')
def create_potential_chunks_for_markdown(tenant_id, document_version, input_file): def create_potential_chunks_for_markdown(tenant_id, document_version, input_file):
try: try:
current_app.logger.info(f'Creating potential chunks for tenant {tenant_id}') current_app.logger.info(f'Creating potential chunks for tenant {tenant_id}')
@@ -328,7 +261,6 @@ def create_potential_chunks_for_markdown(tenant_id, document_version, input_file
md_header_splits = markdown_splitter.split_text(markdown) md_header_splits = markdown_splitter.split_text(markdown)
potential_chunks = [doc.page_content for doc in md_header_splits] potential_chunks = [doc.page_content for doc in md_header_splits]
current_app.logger.debug(f'Created {len(potential_chunks)} potential chunks for tenant {tenant_id}')
return potential_chunks return potential_chunks
except Exception as e: except Exception as e:
current_app.logger.error(f'Error creating potential chunks for tenant {tenant_id}, with error: {e}') current_app.logger.error(f'Error creating potential chunks for tenant {tenant_id}, with error: {e}')
@@ -361,3 +293,69 @@ def combine_chunks_for_markdown(potential_chunks, min_chars, max_chars):
actual_chunks.append(current_chunk) actual_chunks.append(current_chunk)
return actual_chunks return actual_chunks
def get_processor_for_document(catalog_id: int, file_type: str, sub_file_type: str = None) -> Processor:
"""
Get the appropriate processor for a document based on catalog_id, file_type and optional sub_file_type.
Args:
catalog_id: ID of the catalog
file_type: Type of file (e.g., 'pdf', 'html')
sub_file_type: Optional sub-type for specialized processing
Returns:
Processor instance
Raises:
ValueError: If no matching processor is found
"""
try:
# Start with base query for catalog
query = Processor.query.filter_by(catalog_id=catalog_id)
# Find processor type that handles this file type
matching_processor_type = None
for proc_type, config in PROCESSOR_TYPES.items():
supported_types = config['file_types']
if isinstance(supported_types, str):
supported_types = [t.strip() for t in supported_types.split(',')]
if file_type in supported_types:
matching_processor_type = proc_type
break
if not matching_processor_type:
raise ValueError(f"No processor type found for file type: {file_type}")
# Add processor type condition
query = query.filter_by(type=matching_processor_type)
# If sub_file_type is provided, add that condition
if sub_file_type:
query = query.filter_by(sub_file_type=sub_file_type)
else:
# If no sub_file_type, prefer processors without sub_file_type specification
query = query.filter(or_(Processor.sub_file_type.is_(None),
Processor.sub_file_type == ''))
# Get the first matching processor
processor = query.first()
if not processor:
if sub_file_type:
raise ValueError(
f"No processor found for catalog {catalog_id} of type {matching_processor_type}, "
f"file type {file_type}, sub-type {sub_file_type}"
)
else:
raise ValueError(
f"No processor found for catalog {catalog_id}, "
f"file type {file_type}"
)
return processor
except Exception as e:
current_app.logger.error(f"Error finding processor: {str(e)}")
raise

7
integrations/Wordpress/eveai-chat-widget/eveai-chat_plugin.php
View File

@@ -29,7 +29,8 @@ function eveai_chat_shortcode($atts) {
'domain' => '', 'domain' => '',
'language' => 'en', 'language' => 'en',
'supported_languages' => 'en,fr,de,es', 'supported_languages' => 'en,fr,de,es',
'server_url' => 'https://evie.askeveai.com' 'server_url' => 'https://evie.askeveai.com',
'specialist_id' => '1' // Added specialist_id parameter
); );
// Merge provided attributes with defaults // Merge provided attributes with defaults
@@ -42,6 +43,7 @@ function eveai_chat_shortcode($atts) {
$language = sanitize_text_field($atts['language']); $language = sanitize_text_field($atts['language']);
$supported_languages = sanitize_text_field($atts['supported_languages']); $supported_languages = sanitize_text_field($atts['supported_languages']);
$server_url = esc_url_raw($atts['server_url']); $server_url = esc_url_raw($atts['server_url']);
$specialist_id = sanitize_text_field($atts['specialist_id']); // Sanitize specialist_id
// Generate a unique ID for this instance of the chat widget // Generate a unique ID for this instance of the chat widget
$chat_id = 'chat-container-' . uniqid(); $chat_id = 'chat-container-' . uniqid();
@@ -55,7 +57,8 @@ function eveai_chat_shortcode($atts) {
'$domain', '$domain',
'$language', '$language',
'$supported_languages', '$supported_languages',
'$server_url' '$server_url',
'$specialist_id'
); );
eveAI.initializeChat('$chat_id'); eveAI.initializeChat('$chat_id');
}); });

37
integrations/Wordpress/eveai-chat-widget/js/eveai-chat-widget.js
View File

@@ -1,6 +1,6 @@
class EveAIChatWidget extends HTMLElement { class EveAIChatWidget extends HTMLElement {
static get observedAttributes() { static get observedAttributes() {
return ['tenant-id', 'api-key', 'domain', 'language', 'languages', 'server-url']; return ['tenant-id', 'api-key', 'domain', 'language', 'languages', 'server-url', 'specialist-id'];
} }
constructor() { constructor() {
@@ -14,6 +14,7 @@ class EveAIChatWidget extends HTMLElement {
this.maxConnectionIdleTime = 1 * 60 * 60 * 1000; // 1 hours in milliseconds this.maxConnectionIdleTime = 1 * 60 * 60 * 1000; // 1 hours in milliseconds
this.languages = [] this.languages = []
this.room = null; this.room = null;
this.specialistId = null;
console.log('EveAIChatWidget constructor called'); console.log('EveAIChatWidget constructor called');
} }
@@ -89,6 +90,7 @@ class EveAIChatWidget extends HTMLElement {
this.languages = languageAttr ? languageAttr.split(',') : []; this.languages = languageAttr ? languageAttr.split(',') : [];
this.serverUrl = this.getAttribute('server-url'); this.serverUrl = this.getAttribute('server-url');
this.currentLanguage = this.language; this.currentLanguage = this.language;
this.specialistId = this.getAttribute('specialist-id');
console.log('Updated attributes:', { console.log('Updated attributes:', {
tenantId: this.tenantId, tenantId: this.tenantId,
apiKey: this.apiKey, apiKey: this.apiKey,
@@ -96,7 +98,8 @@ class EveAIChatWidget extends HTMLElement {
language: this.language, language: this.language,
currentLanguage: this.currentLanguage, currentLanguage: this.currentLanguage,
languages: this.languages, languages: this.languages,
serverUrl: this.serverUrl serverUrl: this.serverUrl,
specialistId: this.specialistId
}); });
} }
@@ -107,15 +110,17 @@ class EveAIChatWidget extends HTMLElement {
const language = this.getAttribute('language'); const language = this.getAttribute('language');
const languages = this.getAttribute('languages'); const languages = this.getAttribute('languages');
const serverUrl = this.getAttribute('server-url'); const serverUrl = this.getAttribute('server-url');
const specialistId = this.getAttribute('specialist-id')
console.log('Checking if all attributes are set:', { console.log('Checking if all attributes are set:', {
tenantId, tenantId,
apiKey, apiKey,
domain, domain,
language, language,
languages, languages,
serverUrl serverUrl,
specialistId
}); });
return tenantId && apiKey && domain && language && languages && serverUrl; return tenantId && apiKey && domain && language && languages && serverUrl && specialistId;
} }
createLanguageDropdown() { createLanguageDropdown() {
@@ -241,13 +246,13 @@ class EveAIChatWidget extends HTMLElement {
this.socket.on('task_status', (data) => { this.socket.on('task_status', (data) => {
console.log('Task status received:', data.status); console.log('Task status received:', data.status);
console.log('Task ID received:', data.taskId); console.log('Task ID received:', data.taskId);
console.log('Citations type:', typeof data.citations, 'Citations:', data.citations); console.log('Citations type:', typeof data.results.citations, 'Citations:', data.results.citations);
if (data.status === 'pending') { if (data.status === 'pending') {
this.updateProgress(); this.updateProgress();
setTimeout(() => this.checkTaskStatus(data.taskId), 1000); // Poll every second setTimeout(() => this.checkTaskStatus(data.taskId), 1000); // Poll every second
} else if (data.status === 'success') { } else if (data.status === 'success') {
this.addBotMessage(data.answer, data.interaction_id, data.algorithm, data.citations); this.addBotMessage(data.results.answer, data.interaction_id, data.algorithm, data.results.citations || []);
this.clearProgress(); // Clear progress indicator when done this.clearProgress(); // Clear progress indicator when done
} else { } else {
this.setStatusMessage('Failed to process message.'); this.setStatusMessage('Failed to process message.');
@@ -450,15 +455,21 @@ toggleFeedback(thumbsUp, thumbsDown, feedback, interactionId) {
const selectedLanguage = this.languageSelect.value; const selectedLanguage = this.languageSelect.value;
console.log('Sending message to backend'); // Updated message structure to match specialist execution format
this.socket.emit('user_message', { const messageData = {
tenantId: this.tenantId, tenantId: parseInt(this.tenantId),
token: this.jwtToken, token: this.jwtToken,
message, specialistId: parseInt(this.specialistId),
language: selectedLanguage, arguments: {
language: selectedLanguage,
query: message
},
timezone: this.userTimezone timezone: this.userTimezone
}); };
this.setStatusMessage('Processing started ...')
console.log('Sending message to backend:', messageData);
this.socket.emit('user_message', messageData);
this.setStatusMessage('Processing started ...');
} }
toggleSendButton(isProcessing) { toggleSendButton(isProcessing) {

6
integrations/Wordpress/eveai-chat-widget/js/eveai-sdk.js
View File

@@ -1,14 +1,15 @@
// static/js/eveai-sdk.js // static/js/eveai-sdk.js
class EveAI { class EveAI {
constructor(tenantId, apiKey, domain, language, languages, serverUrl) { constructor(tenantId, apiKey, domain, language, languages, serverUrl, specialistId) {
this.tenantId = tenantId; this.tenantId = tenantId;
this.apiKey = apiKey; this.apiKey = apiKey;
this.domain = domain; this.domain = domain;
this.language = language; this.language = language;
this.languages = languages; this.languages = languages;
this.serverUrl = serverUrl; this.serverUrl = serverUrl;
this.specialistId = specialistId;
console.log('EveAI constructor:', { tenantId, apiKey, domain, language, languages, serverUrl }); console.log('EveAI constructor:', { tenantId, apiKey, domain, language, languages, serverUrl, specialistId });
} }
initializeChat(containerId) { initializeChat(containerId) {
@@ -23,6 +24,7 @@ class EveAI {
chatWidget.setAttribute('language', this.language); chatWidget.setAttribute('language', this.language);
chatWidget.setAttribute('languages', this.languages); chatWidget.setAttribute('languages', this.languages);
chatWidget.setAttribute('server-url', this.serverUrl); chatWidget.setAttribute('server-url', this.serverUrl);
chatWidget.setAttribute('specialist-id', this.specialistId);
}); });
} else { } else {
console.error('Container not found'); console.error('Container not found');

31
migrations/tenant/versions/2370a17da7cb_allow_json_specialist_input_and_results_.py Normal file
View File

@@ -0,0 +1,31 @@
"""Allow JSON specialist input and results for interactions
Revision ID: 2370a17da7cb
Revises: 0c347651837c
Create Date: 2024-11-07 14:33:57.678861
"""
from alembic import op
import sqlalchemy as sa
import pgvector
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '2370a17da7cb'
down_revision = '0c347651837c'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('interaction', sa.Column('specialist_arguments', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
op.add_column('interaction', sa.Column('specialist_results', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('interaction', 'specialist_results')
op.drop_column('interaction', 'specialist_arguments')
# ### end Alembic commands ###

31
migrations/tenant/versions/4bf121f340e5_add_sub_file_type_to_processor_and_.py Normal file
View File

@@ -0,0 +1,31 @@
"""Add sub_file_type to processor and document_version
Revision ID: 4bf121f340e5
Revises: e54c830a88c9
Create Date: 2024-11-11 13:03:23.265230
"""
from alembic import op
import sqlalchemy as sa
import pgvector
# revision identifiers, used by Alembic.
revision = '4bf121f340e5'
down_revision = 'e54c830a88c9'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('document_version', sa.Column('sub_file_type', sa.String(length=50), nullable=True))
op.add_column('processor', sa.Column('sub_file_type', sa.String(length=50), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('processor', 'sub_file_type')
op.drop_column('document_version', 'sub_file_type')
# ### end Alembic commands ###

41
migrations/tenant/versions/7872ffdbac6c_removed_obsolete_interaction_fields_.py Normal file
View File

@@ -0,0 +1,41 @@
"""Removed obsolete interaction fields, added specialist relation to interaction
Revision ID: 7872ffdbac6c
Revises: 2370a17da7cb
Create Date: 2024-11-07 15:14:12.263011
"""
from alembic import op
import sqlalchemy as sa
import pgvector
# revision identifiers, used by Alembic.
revision = '7872ffdbac6c'
down_revision = '2370a17da7cb'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('interaction', sa.Column('specialist_id', sa.Integer(), nullable=True))
op.create_foreign_key(None, 'interaction', 'specialist', ['specialist_id'], ['id'])
op.drop_column('interaction', 'question')
op.drop_column('interaction', 'answer')
op.drop_column('interaction', 'algorithm_used')
op.drop_column('interaction', 'language')
op.drop_column('interaction', 'detailed_question')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('interaction', sa.Column('detailed_question', sa.TEXT(), autoincrement=False, nullable=True))
op.add_column('interaction', sa.Column('language', sa.VARCHAR(length=2), autoincrement=False, nullable=False))
op.add_column('interaction', sa.Column('algorithm_used', sa.VARCHAR(length=20), autoincrement=False, nullable=True))
op.add_column('interaction', sa.Column('answer', sa.TEXT(), autoincrement=False, nullable=True))
op.add_column('interaction', sa.Column('question', sa.TEXT(), autoincrement=False, nullable=False))
op.drop_constraint(None, 'interaction', type_='foreignkey')
op.drop_column('interaction', 'specialist_id')
# ### end Alembic commands ###

49
migrations/tenant/versions/95f82a301dff_adding_processor_model.py Normal file
View File

@@ -0,0 +1,49 @@
"""Adding Processor Model
Revision ID: 95f82a301dff
Revises: 7872ffdbac6c
Create Date: 2024-11-11 11:26:23.103689
"""
from alembic import op
import sqlalchemy as sa
import pgvector
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '95f82a301dff'
down_revision = '7872ffdbac6c'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('processor',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=50), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('catalog_id', sa.Integer(), nullable=True),
sa.Column('type', sa.String(length=50), nullable=False),
sa.Column('min_chunk_size', sa.Integer(), nullable=True),
sa.Column('max_chunk_size', sa.Integer(), nullable=True),
sa.Column('tuning', sa.Boolean(), nullable=True),
sa.Column('user_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('system_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('configuration', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
sa.Column('created_by', sa.Integer(), nullable=True),
sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
sa.Column('updated_by', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['catalog_id'], ['catalog.id'], ),
sa.ForeignKeyConstraint(['created_by'], ['public.user.id'], ),
sa.ForeignKeyConstraint(['updated_by'], ['public.user.id'], ),
sa.PrimaryKeyConstraint('id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('processor')
# ### end Alembic commands ###

47
migrations/tenant/versions/e54c830a88c9_remove_obsolete_columns_due_to_.py Normal file
View File

@@ -0,0 +1,47 @@
"""Remove obsolete columns due to Processor model introduction
Revision ID: e54c830a88c9
Revises: 95f82a301dff
Create Date: 2024-11-11 11:31:33.478714
"""
from alembic import op
import sqlalchemy as sa
import pgvector
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'e54c830a88c9'
down_revision = '95f82a301dff'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('catalog', 'chat_RAG_temperature')
op.drop_column('catalog', 'html_included_elements')
op.drop_column('catalog', 'html_tags')
op.drop_column('catalog', 'embed_tuning')
op.drop_column('catalog', 'html_end_tags')
op.drop_column('catalog', 'html_excluded_classes')
op.drop_column('catalog', 'chat_no_RAG_temperature')
op.drop_column('catalog', 'html_excluded_elements')
op.drop_column('processor', 'min_chunk_size')
op.drop_column('processor', 'max_chunk_size')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('processor', sa.Column('max_chunk_size', sa.INTEGER(), autoincrement=False, nullable=True))
op.add_column('processor', sa.Column('min_chunk_size', sa.INTEGER(), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('html_excluded_elements', postgresql.ARRAY(sa.VARCHAR(length=50)), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('chat_no_RAG_temperature', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('html_excluded_classes', postgresql.ARRAY(sa.VARCHAR(length=200)), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('html_end_tags', postgresql.ARRAY(sa.VARCHAR(length=10)), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('embed_tuning', sa.BOOLEAN(), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('html_tags', postgresql.ARRAY(sa.VARCHAR(length=10)), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('html_included_elements', postgresql.ARRAY(sa.VARCHAR(length=50)), autoincrement=False, nullable=True))
op.add_column('catalog', sa.Column('chat_RAG_temperature', sa.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True))
# ### end Alembic commands ###

3
nginx/public/chat_evie.html
View File

@@ -21,7 +21,8 @@
'http://macstudio.ask-eve-ai-local.com', 'http://macstudio.ask-eve-ai-local.com',
'en', 'en',
'en,fr,nl', 'en,fr,nl',
'http://macstudio.ask-eve-ai-local.com:8080/' 'http://macstudio.ask-eve-ai-local.com:8080/',
'7'
); );
eveAI.initializeChat('chat-container'); eveAI.initializeChat('chat-container');
}); });

Some files were not shown because too many files have changed in this diff Show More