Removing DocumentLanguage, as both System Context and User Context are to be defined on DocumentVersion level.

Finetuning of embedding workers.

common/langchain/EveAIRetriever.py

@@ -0,0 +1,50 @@
+from langchain_core.retrievers import BaseRetriever
+from sqlalchemy.exc import SQLAlchemyError
+from pydantic import BaseModel, Field
+from typing import Any, Dict
+
+from common.extensions import db
+from flask import current_app
+from config.logging_config import LOGGING
+
+
+class EveAIRetriever(BaseRetriever):
+    model_variables: Dict[str, Any] = Field(...)
+
+    def __init__(self, model_variables: Dict[str, Any]):
+        super().__init__()
+        current_app.logger.debug('Initializing EveAIRetriever')
+        self.model_variables = model_variables
+        current_app.logger.debug('EveAIRetriever initialized')
+
+    def _get_relevant_documents(self, query: str):
+        current_app.logger.debug(f'Retrieving relevant documents for query: {query}')
+        query_embedding = self._get_query_embedding(query)
+        db_class = self.model_variables['embedding_db_model']
+        similarity_threshold = self.model_variables['similarity_threshold']
+        k = self.model_variables['k']
+        try:
+            res = (
+                db.session.query(db_class,
+                                 db_class.embedding.cosine_distance(query_embedding)
+                                 .label('distance'))
+                .filter(db_class.embedding.cosine_distance(query_embedding) < similarity_threshold)
+                .order_by('distance')
+                .limit(k)
+                .all()
+            )
+            current_app.rag_tuning_logger.debug(f'Retrieved {len(res)} relevant documents')
+            current_app.rag_tuning_logger.debug(f'---------------------------------------')
+            for doc in res:
+                current_app.rag_tuning_logger.debug(f'Document ID: {doc[0].id} - Distance: {doc[1]}\n')
+                current_app.rag_tuning_logger.debug(f'Chunk: \n {doc[0].chunk}\n\n')
+
+        except SQLAlchemyError as e:
+            current_app.logger.error(f'Error retrieving relevant documents: {e}')
+            return []
+        return res
+
+    def _get_query_embedding(self, query: str):
+        embedding_model = self.model_variables['embedding_model']
+        query_embedding = embedding_model.embed_query(query)
+        return query_embedding

common/models/document.py

@@ -17,50 +17,22 @@ class Document(db.Model):
     updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
 
     # Relations
-    languages = db.relationship('DocumentLanguage', backref='document', lazy=True)
+    versions = db.relationship('DocumentVersion', backref='document', lazy=True)
 
     def __repr__(self):
         return f"<Document {self.id}: {self.name}>"
 
 
-class DocumentLanguage(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
-    document_id = db.Column(db.Integer, db.ForeignKey(Document.id), nullable=False)
-    language = db.Column(db.String(2), nullable=False)
-    user_context = db.Column(db.Text, nullable=True)
-    system_context = db.Column(db.Text, nullable=True)
-    latest_version_id = db.Column(db.Integer, db.ForeignKey('document_version.id'), nullable=True)
-
-    # Versioning Information
-    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
-    created_by = db.Column(db.Integer, db.ForeignKey(User.id), nullable=False)
-    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
-    updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
-
-    # Relations
-    versions = db.relationship(
-        'DocumentVersion',
-        backref='document_language',
-        lazy='joined',
-        foreign_keys='DocumentVersion.doc_lang_id'
-    )
-    latest_version = db.relationship(
-        'DocumentVersion',
-        uselist=False,
-        foreign_keys=[latest_version_id]
-    )
-
-    def __repr__(self):
-        return f"<DocumentLanguage {self.document_id}.{self.language}>"
-
-
 class DocumentVersion(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    doc_lang_id = db.Column(db.Integer, db.ForeignKey(DocumentLanguage.id), nullable=False)
+    doc_id = db.Column(db.Integer, db.ForeignKey(Document.id), nullable=False)
     url = db.Column(db.String(200), nullable=True)
     file_location = db.Column(db.String(255), nullable=True)
     file_name = db.Column(db.String(200), nullable=True)
     file_type = db.Column(db.String(20), nullable=True)
+    language = db.Column(db.String(2), nullable=False)
+    user_context = db.Column(db.Text, nullable=True)
+    system_context = db.Column(db.Text, nullable=True)
 
     # Versioning Information
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now())
@@ -81,7 +53,7 @@ class DocumentVersion(db.Model):
         return f"<DocumentVersion {self.document_language.document_id}.{self.document_language.language}>.{self.id}>"
 
     def calc_file_location(self):
-        return f"{self.document_language.document.tenant_id}/{self.document_language.document.id}/{self.document_language.language}"
+        return f"{self.document.tenant_id}/{self.document.id}/{self.language}"
 
     def calc_file_name(self):
         return f"{self.id}.{self.file_type}"

common/models/user.py

@@ -48,6 +48,10 @@ class Tenant(db.Model):
     allowed_monthly_interactions = db.Column(db.Integer, nullable=True)
     encrypted_chat_api_key = db.Column(db.String(500), nullable=True)
 
+    # Tuning enablers
+    embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
+    rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
+
     # Relations
     users = db.relationship('User', backref='tenant')
     domains = db.relationship('TenantDomain', backref='tenant')
@@ -133,7 +137,10 @@ class TenantDomain(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
-    domain = db.Column(db.String(255), unique=True, nullable=False)
+    # Originally, domain was required to be unique.
+    # However, several tenants can run from the same domain (e.g. for demo purposes,
+    # but also internal and external chat clients.
+    domain = db.Column(db.String(255), nullable=False)
     valid_to = db.Column(db.Date, nullable=True)
 
     # Versioning Information

common/utils/model_utils.py

@@ -2,6 +2,7 @@ from flask import current_app
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
+import ast
 
 from common.models.document import EmbeddingSmallOpenAI
 
@@ -35,6 +36,23 @@ def select_model_variables(tenant):
     else:
         model_variables['no_RAG_temperature'] = 0.5
 
+    # Set Tuning variables
+    if tenant.embed_tuning:
+        model_variables['embed_tuning'] = tenant.embed_tuning
+    else:
+        model_variables['embed_tuning'] = False
+
+    if tenant.rag_tuning:
+        model_variables['rag_tuning'] = tenant.rag_tuning
+    else:
+        model_variables['rag_tuning'] = False
+
+    # Set HTML Chunking Variables
+    model_variables['html_tags'] = tenant.html_tags
+    model_variables['html_end_tags'] = tenant.html_end_tags
+    model_variables['html_included_elements'] = tenant.html_included_elements
+    model_variables['html_excluded_elements'] = tenant.html_excluded_elements
+
     # Set Embedding variables
     match embedding_provider:
         case 'openai':