Removing DocumentLanguage, as both System Context and User Context are to be defined on DocumentVersion level.

Finetuning of embedding workers.

common/utils/model_utils.py

@@ -2,6 +2,7 @@ from flask import current_app
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
+import ast
 
 from common.models.document import EmbeddingSmallOpenAI
 
@@ -35,6 +36,23 @@ def select_model_variables(tenant):
     else:
         model_variables['no_RAG_temperature'] = 0.5
 
+    # Set Tuning variables
+    if tenant.embed_tuning:
+        model_variables['embed_tuning'] = tenant.embed_tuning
+    else:
+        model_variables['embed_tuning'] = False
+
+    if tenant.rag_tuning:
+        model_variables['rag_tuning'] = tenant.rag_tuning
+    else:
+        model_variables['rag_tuning'] = False
+
+    # Set HTML Chunking Variables
+    model_variables['html_tags'] = tenant.html_tags
+    model_variables['html_end_tags'] = tenant.html_end_tags
+    model_variables['html_included_elements'] = tenant.html_included_elements
+    model_variables['html_excluded_elements'] = tenant.html_excluded_elements
+
     # Set Embedding variables
     match embedding_provider:
         case 'openai':