Removing DocumentLanguage, as both System Context and User Context are to be defined on DocumentVersion level.

Finetuning of embedding workers.

eveai_app/templates/document/document_languages.html

@@ -1,24 +0,0 @@
-{% extends 'base.html' %}
-{%  from 'macros.html' import render_selectable_table, render_pagination %}
-
-{% block title %}Document Languages{% endblock %}
-
-{% block content_title %}Document Languages{% endblock %}
-{% block content_description %}View Languages for {{ document }}{% endblock %}
-{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto">{% endblock %}
-
-{% block content %}
-<div class="container">
-    <form method="POST" action="{{ url_for('document_bp.handle_document_language_selection') }}">
-        {{ render_selectable_table(headers=["Document Language ID", "Language", "User Context", "System Context"], rows=rows, selectable=True, id="documentsTable") }}
-        <div class="form-group mt-3">
-            <button type="submit" name="action" value="edit_document_language" class="btn btn-primary">Edit Document Language</button>
-            <button type="submit" name="action" value="document_versions" class="btn btn-secondary">Show Document Versions</button>
-        </div>
-    </form>
-</div>
-{% endblock %}
-
-{% block content_footer %}
-    {{ render_pagination(pagination, 'document_bp.documents') }}
-{% endblock %}

eveai_app/templates/document/document_versions.html

@@ -12,7 +12,8 @@
     <form method="POST" action="{{ url_for('document_bp.handle_document_version_selection') }}">
         {{ render_selectable_table(headers=["Document Version ID", "URL", "File Location", "File Name", "File Type", "Processing", "Processing Start", "Proceeing Finish"], rows=rows, selectable=True, id="versionsTable") }}
         <div class="form-group mt-3">
-            <button type="submit" name="action" value="process_document_version" class="btn btn-primary">Process Document Version</button>
+            <button type="submit" name="action" value="edit_document_version" class="btn btn-primary">Edit Document Version</button>
+            <button type="submit" name="action" value="process_document_version" class="btn btn-secondary">Process Document Version</button>
         </div>
     </form>
 </div>

eveai_app/templates/document/documents.html

@@ -13,7 +13,7 @@
         {{ render_selectable_table(headers=["Document ID", "Name", "Valid From", "Valid To"], rows=rows, selectable=True, id="documentsTable") }}
         <div class="form-group mt-3">
             <button type="submit" name="action" value="edit_document" class="btn btn-primary">Edit Document</button>
-            <button type="submit" name="action" value="document_languages" class="btn btn-secondary">Show Document Languages</button>
+            <button type="submit" name="action" value="document_versions" class="btn btn-secondary">Show Document Versions</button>
         </div>
     </form>
 </div>

eveai_app/templates/navbar.html

@@ -72,6 +72,7 @@
                                     {'name': 'Tenant List', 'url': '/user/select_tenant', 'roles': ['Super User']},
                                     {'name': 'Tenant Registration', 'url': '/user/tenant', 'roles': ['Super User']},
                                     {'name': 'Tenant Overview', 'url': '/user/tenant_overview', 'roles': ['Super User', 'Tenant Admin']},
+                                    {'name': 'Edit Tenant', 'url': '/user/tenant/' ~ session['tenant'].get('id'), 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Tenant Domains', 'url': '/user/view_tenant_domains', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Tenant Domain Registration', 'url': '/user/tenant_domain', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'User List', 'url': '/user/view_users', 'roles': ['Super User', 'Tenant Admin']},

eveai_app/templates/user/tenant_overview.html

@@ -40,6 +40,11 @@
                                 Embedding Search
                             </a>
                         </li>
+                        <li class="nav-item">
+                            <a class="nav-link mb-0 px-0 py-1" data-toggle="tab" href="#tuning-tab" role="tab" aria-controls="html-chunking" aria-selected="false">
+                                Tuning
+                            </a>
+                        </li>
                     </ul>
                 </div>
                 <div class="tab-content tab-space">
@@ -73,6 +78,13 @@
                             {{ render_included_field(field, disabled_fields=es_fields, include_fields=es_fields) }}
                         {% endfor %}
                     </div>
+                    <!-- Tuning Settings Tab -->
+                    <div class="tab-pane fade" id="tuning-tab" role="tabpanel">
+                        {% set tuning_fields = ['embed_tuning', 'rag_tuning', ] %}
+                        {% for field in form %}
+                            {{ render_included_field(field, disabled_fields=tuning_fields, include_fields=tuning_fields) }}
+                        {% endfor %}
+                    </div>
                 </div>
             </div>
         </div>

eveai_app/views/document_forms.py

@@ -47,7 +47,7 @@ class EditDocumentForm(FlaskForm):
     submit = SubmitField('Submit')
 
 
-class EditDocumentLanguageForm(FlaskForm):
+class EditDocumentVersionForm(FlaskForm):
     language = StringField('Language')
     user_context = TextAreaField('User Context', validators=[Optional()])
     system_context = TextAreaField('System Context', validators=[Optional()])

eveai_app/views/document_views.py

@@ -13,9 +13,9 @@ from requests.exceptions import SSLError
 from urllib.parse import urlparse
 import io
 
-from common.models.document import Document, DocumentLanguage, DocumentVersion
+from common.models.document import Document, DocumentVersion
 from common.extensions import db
-from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentLanguageForm
+from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm
 from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
@@ -59,7 +59,7 @@ def add_document():
         filename = secure_filename(file.filename)
         extension = filename.rsplit('.', 1)[1].lower()
 
-        new_doc, new_doc_lang, new_doc_vers = create_document_stack(form, file, filename, extension)
+        new_doc, new_doc_vers = create_document_stack(form, file, filename, extension)
 
         task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
             session['tenant']['id'],
@@ -100,7 +100,7 @@ def add_url():
             filename += '.html'
         extension = 'html'
 
-        new_doc, new_doc_lang, new_doc_vers = create_document_stack(form, file, filename, extension)
+        new_doc, new_doc_vers = create_document_stack(form, file, filename, extension)
 
         task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
             session['tenant']['id'],
@@ -146,8 +146,8 @@ def handle_document_selection():
     match action:
         case 'edit_document':
             return redirect(prefixed_url_for('document_bp.edit_document', document_id=doc_id))
-        case 'document_languages':
-            return redirect(prefixed_url_for('document_bp.document_languages', document_id=doc_id))
+        case 'document_versions':
+            return redirect(prefixed_url_for('document_bp.document_versions', document_id=doc_id))
 
     # Add more conditions for other actions
     return redirect(prefixed_url_for('document_bp.documents'))
@@ -180,81 +180,46 @@ def edit_document(document_id):
     return render_template('document/edit_document.html', form=form, document_id=document_id)
 
 
-@document_bp.route('/document_languages/<int:document_id>', methods=['GET', 'POST'])
+@document_bp.route('/edit_document_version/<int:document_version_id>', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
-def document_languages(document_id):
-    doc = Document.query.get_or_404(document_id)
-    doc_desc = f'Document {doc.id}: {doc.name}'
-    page = request.args.get('page', 1, type=int)
-    per_page = request.args.get('per_page', 10, type=int)
-
-    query = DocumentLanguage.query.filter_by(document_id=document_id).order_by(DocumentLanguage.language)
-
-    pagination = query.paginate(page=page, per_page=per_page, error_out=False)
-    doc_langs = pagination.items
-
-    rows = prepare_table_for_macro(doc_langs, [('id', ''), ('language', ''), ('user_context', ''),
-                                               ('system_context', '')])
-
-    return render_template('document/document_languages.html', rows=rows, pagination=pagination, document=doc_desc)
-
-
-@document_bp.route('/handle_document_language_selection', methods=['POST'])
-@roles_accepted('Super User', 'Tenant Admin')
-def handle_document_language_selection():
-    document_language_identification = request.form['selected_row']
-    doc_lang_id = ast.literal_eval(document_language_identification).get('value')
-
-    action = request.form['action']
-
-    match action:
-        case 'edit_document_language':
-            return redirect(prefixed_url_for('document_bp.edit_document_language', document_language_id=doc_lang_id))
-        case 'document_versions':
-            return redirect(prefixed_url_for('document_bp.document_versions', document_language_id=doc_lang_id))
-
-    # Add more conditions for other actions
-    return redirect(prefixed_url_for('document_bp.document_languages'))
-
-
-@document_bp.route('/edit_document_language/<int:document_language_id>', methods=['GET', 'POST'])
-@roles_accepted('Super User', 'Tenant Admin')
-def edit_document_language(document_language_id):
-    doc_lang = DocumentLanguage.query.get_or_404(document_language_id)
-    form = EditDocumentLanguageForm(obj=doc_lang)
+def edit_document_version(document_version_id):
+    doc_vers = DocumentVersion.query.get_or_404(document_version_id)
+    form = EditDocumentVersionForm(obj=doc_vers)
 
     if form.validate_on_submit():
-        doc_lang.user_context = form.user_context.data
+        doc_vers.user_context = form.user_context.data
 
-        update_logging_information(doc_lang, dt.now(tz.utc))
+        update_logging_information(doc_vers, dt.now(tz.utc))
 
         try:
-            db.session.add(doc_lang)
+            db.session.add(doc_vers)
             db.session.commit()
-            flash(f'Document Language {doc_lang.id} updated successfully', 'success')
+            flash(f'Document Version {doc_vers.id} updated successfully', 'success')
         except SQLAlchemyError as e:
             db.session.rollback()
-            flash(f'Error updating document language: {e}', 'danger')
-            current_app.logger.error(f'Error updating document language {doc_lang.id} '
+            flash(f'Error updating document version: {e}', 'danger')
+            current_app.logger.error(f'Error updating document version {doc_vers.id} '
                                      f'for tenant {session['tenant']['id']}: {e}')
     else:
         form_validation_failed(request, form)
 
-    return render_template('document/edit_document_language.html', form=form, document_langauge_id=document_language_id,
-                           doc_details=f'Document {doc_lang.document.name}')
+    return render_template('document/edit_document_version.html', form=form, document_version_id=document_version_id,
+                           doc_details=f'Document {doc_vers.document.name}')
 
 
-@document_bp.route('/document_versions/<int:document_language_id>', methods=['GET', 'POST'])
+@document_bp.route('/document_versions/<int:document_id>', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
-def document_versions(document_language_id):
+def document_versions(document_id):
     flash(f'Processing documents is a long running process. Please be careful retriggering processing!', 'danger')
-    doc_lang = DocumentLanguage.query.get_or_404(document_language_id)
-    doc_desc = f'Document {doc_lang.document.name}, Language {doc_lang.language}'
+    doc_vers = DocumentVersion.query.get_or_404(document_id)
+    doc_desc = f'Document {doc_vers.document.name}, Language {doc_vers.language}'
 
     page = request.args.get('page', 1, type=int)
     per_page = request.args.get('per_page', 10, type=int)
 
-    query = DocumentVersion.query.filter_by(doc_lang_id=document_language_id).order_by(desc(DocumentVersion.id))
+    query = (DocumentVersion.query.filter_by(doc_id=document_id)
+             .order_by(DocumentVersion.language)
+             .order_by(desc(DocumentVersion.id)))
 
     pagination = query.paginate(page=page, per_page=per_page, error_out=False)
     doc_langs = pagination.items
@@ -276,6 +241,8 @@ def handle_document_version_selection():
     action = request.form['action']
 
     match action:
+        case 'edit_document_version':
+            return redirect(prefixed_url_for('document_bp.edit_document_version', document_version_id=doc_vers_id))
         case 'process_document_version':
             process_version(doc_vers_id)
             # Add more conditions for other actions
@@ -315,17 +282,11 @@ def create_document_stack(form, file, filename, extension):
     # Create the Document
     new_doc = create_document(form, filename)
 
-    # Create the DocumentLanguage
-    new_doc_lang = create_language_for_document(new_doc, form.language.data, form.user_context.data)
-
     # Create the DocumentVersion
-    new_doc_vers = DocumentVersion()
-    new_doc_vers.document_language = new_doc_lang
-    set_logging_information(new_doc_vers, dt.now(tz.utc))
+    new_doc_vers = create_version_for_document(new_doc, form.language.data, form.user_context.data)
 
     try:
         db.session.add(new_doc)
-        db.session.add(new_doc_lang)
         db.session.add(new_doc_vers)
         db.session.commit()
     except SQLAlchemyError as e:
@@ -338,30 +299,12 @@ def create_document_stack(form, file, filename, extension):
         current_app.logger.error('Unknown error')
         raise
 
-    try:
-        new_doc_lang = db.session.merge(new_doc_lang)
-        new_doc_vers = db.session.merge(new_doc_vers)
-        new_doc_lang.latest_version_id = new_doc_vers.id
-        db.session.commit()
-    except SQLAlchemyError as e:
-        current_app.logger.error(f'Error adding document for tenant {session["tenant"]["id"]}: {e}')
-        flash('Error adding document.', 'error')
-        db.session.rollback()
-        error = e.args
-        raise
-    except Exception as e:
-        current_app.logger.error(f'Error adding document for tenant {session["tenant"]["id"]}: {e}')
-        flash('Error adding document.', 'error')
-        db.session.rollback()
-        error = e.args
-        raise
-
     current_app.logger.info(f'Document added successfully for tenant {session["tenant"]["id"]}, '
                             f'Document Version {new_doc.id}')
 
     upload_file_for_version(new_doc_vers, file, extension)
 
-    return new_doc, new_doc_lang, new_doc_vers
+    return new_doc, new_doc_vers
 
 
 def log_session_state(session, msg=""):
@@ -386,21 +329,21 @@ def create_document(form, filename):
     return new_doc
 
 
-def create_language_for_document(document, language, user_context):
-    new_doc_lang = DocumentLanguage()
+def create_version_for_document(document, language, user_context):
+    new_doc_vers = DocumentVersion()
     if language == '':
-        new_doc_lang.language = session['default_language']
+        new_doc_vers.language = session['default_language']
     else:
-        new_doc_lang.language = language
+        new_doc_vers.language = language
 
     if user_context != '':
-        new_doc_lang.user_context = user_context
+        new_doc_vers.user_context = user_context
 
-    new_doc_lang.document = document
+    new_doc_vers.document = document
 
-    set_logging_information(new_doc_lang, dt.now(tz.utc))
+    set_logging_information(new_doc_vers, dt.now(tz.utc))
 
-    return new_doc_lang
+    return new_doc_vers
 
 
 def upload_file_for_version(doc_vers, file, extension):

eveai_app/views/user_forms.py

@@ -33,6 +33,9 @@ class TenantForm(FlaskForm):
     es_similarity_threshold = FloatField('Similarity Threshold for Searching Embeddings (0.5)',
                                          default=0.5,
                                          validators=[NumberRange(min=0, max=1)])
+    # Tuning variables
+    embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
+    rag_tuning = BooleanField('Enable RAG Tuning', default=False)
 
     submit = SubmitField('Submit')
 

eveai_app/views/user_views.py

@@ -53,7 +53,9 @@ def tenant():
                             llm_model=form.llm_model.data,
                             license_start_date=form.license_start_date.data,
                             license_end_date=form.license_end_date.data,
-                            allowed_monthly_interactions=form.allowed_monthly_interactions.data)
+                            allowed_monthly_interactions=form.allowed_monthly_interactions.data,
+                            embed_tuning=form.embed_tuning.data,
+                            rag_tuning=form.rag_tuning.data)
 
         # Handle Embedding Variables
         new_tenant.html_tags = form.html_tags.data.split(',') if form.html_tags.data else []