- Adding a Tenant Type

- Allow filtering on Tenant Types & searching for parts of Tenant names - Implement health checks - Start Prometheus monitoring (needs to be finalized) - Refine audio_processor and srt_processor to reduce duplicate code and support for larger files - Introduce repopack to reason in LLMs about the code
2024-09-13 15:43:40 +02:00
parent 9e14824249
commit 6cf660e622
41 changed files with 687 additions and 579 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,4 @@ migrations/.DS_Store
 migrations/public/.DS_Store
 scripts/.DS_Store
 scripts/__pycache__/run_eveai_app.cpython-312.pyc
+/eveai_repo.txt
--- a/.repopackignore
+++ b/.repopackignore
@@ -0,0 +1,21 @@
+# Add patterns to ignore here, one per line
+# Example:
+# *.log
+# tmp/
+logs/
+nginx/static/assets/fonts/
+nginx/static/assets/img/
+nginx/static/assets/js/
+nginx/static/scss/
+patched_packages/
+migrations/
+*material*
+*nucleo*
+*package*
+nginx/mime.types
+*.gitignore*
+.python-version
+.repopackignore
+repopack.config.json
+
+
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,7 +24,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ### Security
 - In case of vulnerabilities.
- 
+
+## [1.0.7-alfa] - 2024-09-12
+
+### Added
+- Full Document API allowing for creation, updating and invalidation of documents.
+- Metadata fields (JSON) added to DocumentVersion, allowing end-users to add structured information
+- Wordpress plugin eveai_sync to synchronize Wordpress content with EveAI
+
+### Fixed
+- Maximal deduplication of code between views and api in document_utils.py
+
 ## [1.0.6-alfa] - 2024-09-03

 ### Fixed
--- a/common/extensions.py
+++ b/common/extensions.py
@@ -10,8 +10,8 @@ from flask_jwt_extended import JWTManager
 from flask_session import Session
 from flask_wtf import CSRFProtect
 from flask_restx import Api
+from prometheus_flask_exporter import PrometheusMetrics

-from .utils.nginx_utils import prefixed_url_for
 from .utils.simple_encryption import SimpleEncryption
 from .utils.minio_utils import MinioClient

@@ -31,3 +31,4 @@ session = Session()
 api_rest = Api()
 simple_encryption = SimpleEncryption()
 minio_client = MinioClient()
+metrics = PrometheusMetrics.for_app_factory()
--- a/common/models/user.py
+++ b/common/models/user.py
@@ -21,6 +21,7 @@ class Tenant(db.Model):
    website = db.Column(db.String(255), nullable=True)
    timezone = db.Column(db.String(50), nullable=True, default='UTC')
    rag_context = db.Column(db.Text, nullable=True)
+    type = db.Column(db.String(20), nullable=True, server_default='Active')

    # language information
    default_language = db.Column(db.String(2), nullable=True)
@@ -56,7 +57,6 @@ class Tenant(db.Model):
    encrypted_chat_api_key = db.Column(db.String(500), nullable=True)
    encrypted_api_key = db.Column(db.String(500), nullable=True)

-
    # Tuning enablers
    embed_tuning = db.Column(db.Boolean, nullable=True, default=False)
    rag_tuning = db.Column(db.Boolean, nullable=True, default=False)
@@ -75,6 +75,7 @@ class Tenant(db.Model):
            'website': self.website,
            'timezone': self.timezone,
            'rag_context': self.rag_context,
+            'type': self.type,
            'default_language': self.default_language,
            'allowed_languages': self.allowed_languages,
            'embedding_model': self.embedding_model,
--- a/common/utils/model_utils.py
+++ b/common/utils/model_utils.py
@@ -147,10 +147,10 @@ def select_model_variables(tenant):
            match llm_model:
                case 'gpt-4o' | 'gpt-4o-mini':
                    tool_calling_supported = True
-                    PDF_chunk_size = 10000
-                    PDF_chunk_overlap = 200
-                    PDF_min_chunk_size = 8000
-                    PDF_max_chunk_size = 12000
+                    processing_chunk_size = 10000
+                    processing_chunk_overlap = 200
+                    processing_min_chunk_size = 8000
+                    processing_max_chunk_size = 12000
                case _:
                    raise Exception(f'Error setting model variables for tenant {tenant.id} '
                                    f'error: Invalid chat model')
@@ -165,18 +165,18 @@ def select_model_variables(tenant):
                                                          model=llm_model_ext,
                                                          temperature=model_variables['RAG_temperature'])
            tool_calling_supported = True
-            PDF_chunk_size = 10000
-            PDF_chunk_overlap = 200
-            PDF_min_chunk_size = 8000
-            PDF_max_chunk_size = 12000
+            processing_chunk_size = 10000
+            processing_chunk_overlap = 200
+            processing_min_chunk_size = 8000
+            processing_max_chunk_size = 12000
        case _:
            raise Exception(f'Error setting model variables for tenant {tenant.id} '
                            f'error: Invalid chat provider')

-    model_variables['PDF_chunk_size'] = PDF_chunk_size
-    model_variables['PDF_chunk_overlap'] = PDF_chunk_overlap
-    model_variables['PDF_min_chunk_size'] = PDF_min_chunk_size
-    model_variables['PDF_max_chunk_size'] = PDF_max_chunk_size
+    model_variables['processing_chunk_size'] = processing_chunk_size
+    model_variables['processing_chunk_overlap'] = processing_chunk_overlap
+    model_variables['processing_min_chunk_size'] = processing_min_chunk_size
+    model_variables['processing_max_chunk_size'] = processing_max_chunk_size

    if tool_calling_supported:
        model_variables['cited_answer_cls'] = CitedAnswer
--- a/config/config.py
+++ b/config/config.py
@@ -139,7 +139,7 @@ class Config(object):

    SUPPORTED_FILE_TYPES = ['pdf', 'html', 'md', 'txt', 'mp3', 'mp4', 'ogg', 'srt']

-
+    TENANT_TYPES = ['Active', 'Demo', 'Inactive', 'Test']


 class DevConfig(Config):
--- a/config/gc_sa_eveai.json
+++ b/config/gc_sa_eveai.json
@@ -1,13 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "eveai-420711",
-  "private_key_id": "e666408e75793321a6134243628346722a71b3a6",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaGTXCWpq08YD1\nOW4z+gncOlB7T/EIiEwsZgMp6pyUrNioGfiI9YN+uVR0nsUSmFf1YyerRgX7RqD5\nRc7T/OuX8iIvmloK3g7CaFezcVrjnBKcg/QsjDAt/OO3DTk4vykDlh/Kqxx73Jdv\nFH9YSV2H7ToWqIE8CTDnqe8vQS7Bq995c9fPlues31MgndRFg3CFkH0ldfZ4aGm3\n1RnBDyC+9SPQW9e7CJgNN9PWTmOT51Zyy5IRuV5OWePMQaGLVmCo5zNc/EHZEVRu\n1hxJPHL3NNmkYDY8tye8uHgjsAkv8QuwIuUSqnqjoo1/Yg+P0+9GCpePOAJRNxJS\n0YpDFWc5AgMBAAECggEACIU4/hG+bh97BD7JriFhfDDT6bg7g+pCs/hsAlxQ42jv\nOH7pyWuHJXGf5Cwx31usZAq4fcrgYnVpnyl8odIL628y9AjdI66wMuWhZnBFGJgK\nRhHcZWjW8nlXf0lBjwwFe4edzbn1AuWT5fYZ2HWDW2mthY/e8sUwqWPcWsjdifhz\nNR7V+Ia47McKXYgEKjyEObSP1NUOW24zH0DgxS52YPMwa1FoHn6+9Pr8P3TsTSO6\nh6f8tnd81DGl1UH4F5Bj/MHsQXyAMJbu44S4+rZ4Qlk+5xPp9hfCNpxWaHLIkJCg\nYXnC8UAjjyXiqyK0U0RjJf8TS1FxUI4iPepLNqp/pQKBgQDTicZnWFXmCFTnycWp\n66P3Yx0yvlKdUdfnoD/n9NdmUA3TZUlEVfb0IOm7ZFubF/zDTH87XrRiD/NVDbr8\n6bdhA1DXzraxhbfD36Hca6K74Ba4aYJsSWWwI0hL3FDSsv8c7qAIaUF2iwuHb7Y0\nRDcvZqowtQobcQC8cHLc/bI/ZwKBgQC6fMeGaU+lP6jhp9Nb/3Gz5Z1zzCu34IOo\nlgpTNZsowRKYLtjHifrEFi3XRxPKz5thMuJFniof5U4WoMYtRXy+PbgySvBpCia2\nXty05XssnLLMvLpYU5sbQvmOTe20zaIzLohRvvmqrydYIKu62NTubNeuD1L+Zr0q\nz1P5/wUgXwKBgQCW9MrRFQi3j1qHzkVwbOglsmUzwP3TpoQclw8DyIWuTZKQOMeA\nLJh+vr4NLCDzHLsT45MoGv0+vYM4PwQhV+e1I1idqLZXGMV60iv/0A/hYpjUIPch\nr38RoxwEhsRml7XWP7OUTQiaP7+Kdv3fbo6zFOB+wbLkwk90KgrOCX0aIQKBgFeK\n7esmErJjMPdFXk3om0q09nX+mWNHLOb+EDjBiGXYRM9V5oO9PQ/BzaEqh5sEXE+D\noH7H4cR5U3AB5yYnYYi41ngdf7//eO7Rl1AADhOCN9kum1eNX9mrVhU8deMTSRo3\ntNyTBwbeFF0lcRhUY5jNVW4rWW19cz3ed/B6i8CHAoGBAJ/l5rkV74Z5hg6BWNfQ\nYAg/4PLZmjnXIy5QdnWc/PYgbhn5+iVUcL9fSofFzJM1rjFnNcs3S90MGeOmfmo4\nM1WtcQFQbsCGt6+G5uEL/nf74mKUGpOqEM/XSkZ3inweWiDk3LK3iYfXCMBFouIr\n80IlzI1yMf7MVmWn3e1zPjCA\n-----END PRIVATE KEY-----\n",
-  "client_email": "eveai-349@eveai-420711.iam.gserviceaccount.com",
-  "client_id": "109927035346319712442",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/eveai-349%40eveai-420711.iam.gserviceaccount.com",
-  "universe_domain": "googleapis.com"
-}
--- a/config/prompts/openai/gpt-4o.yaml
+++ b/config/prompts/openai/gpt-4o.yaml
@@ -65,11 +65,13 @@ encyclopedia: |

 transcript: |
  You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material.
+  You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk.
+

  # Best practices and steps are:
  - Respect wordings and language(s) used in the transcription. Main language is {language}.
  - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking.    
-  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part.
+  - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences.
  - annotate the text to identify these logical parts using headings in {language}.
  - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription.

@@ -77,4 +79,6 @@ transcript: |

  The transcript is between triple backquotes.

+  $$${previous_part}$$$
+
  ```{transcript}```
--- a/docker/compose_dev.yaml
+++ b/docker/compose_dev.yaml
@@ -96,12 +96,11 @@ services:
       minio:
         condition: service_healthy
    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-#    entrypoint: ["scripts/entrypoint.sh"]
-#    command: ["scripts/start_eveai_app.sh"]
+      test: ["CMD", "curl", "-f", "http://localhost:5001/healthz/ready"]
+      interval: 30s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
    networks:
      - eveai-network

@@ -113,8 +112,6 @@ services:
      platforms:
        - linux/amd64
        - linux/arm64
-#    ports:
-#      - 5001:5001
    environment:
      <<: *common-variables
      COMPONENT_NAME: eveai_workers
@@ -132,13 +129,6 @@ services:
        condition: service_healthy
      minio:
        condition: service_healthy
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
    networks:
      - eveai-network

@@ -168,12 +158,11 @@ services:
      redis:
        condition: service_healthy
    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ]  # Adjust based on your health endpoint
-      interval: 10s
-      timeout: 5s
-      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
+      test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ]  # Adjust based on your health endpoint
+      interval: 30s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
    networks:
      - eveai-network

@@ -185,8 +174,6 @@ services:
      platforms:
        - linux/amd64
        - linux/arm64
-#    ports:
-#      - 5001:5001
    environment:
      <<: *common-variables
      COMPONENT_NAME: eveai_chat_workers
@@ -202,13 +189,6 @@ services:
        condition: service_healthy
      redis:
        condition: service_healthy
-#    healthcheck:
-#      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
-#      interval: 10s
-#      timeout: 5s
-#      retries: 5
-#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
-#    command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
    networks:
      - eveai-network

@@ -240,12 +220,11 @@ services:
      minio:
        condition: service_healthy
    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:5003/health" ]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    #    entrypoint: ["scripts/entrypoint.sh"]
-    #    command: ["scripts/start_eveai_api.sh"]
+      test: [ "CMD", "curl", "-f", "http://localhost:5003/healthz/ready" ]
+      interval: 30s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
    networks:
      - eveai-network

--- a/eveai_api/init.py
+++ b/eveai_api/init.py
@@ -39,9 +39,12 @@ def create_app(config_file=None):
    # Register Necessary Extensions
    register_extensions(app)

-    # register Blueprints
+    # register Namespaces
    register_namespaces(api_rest)

+    # Register Blueprints
+    register_blueprints(app)
+
    # Error handler for the API
    @app.errorhandler(EveAIException)
    def handle_eveai_exception(error):
@@ -102,3 +105,9 @@ def register_extensions(app):
 def register_namespaces(app):
    api_rest.add_namespace(document_ns, path='/api/v1/documents')
    api_rest.add_namespace(auth_ns, path='/api/v1/auth')
+
+
+def register_blueprints(app):
+    from .views.healthz_views import healthz_bp
+    app.register_blueprint(healthz_bp)
+
--- a/eveai_api/views/healthz_views.py
+++ b/eveai_api/views/healthz_views.py
@@ -0,0 +1,82 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+# Define Prometheus metrics
+api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint'])
+api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        "minio": check_minio(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('tasks.ping', queue='embeddings')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+def check_minio():
+    try:
+        # List buckets to check if MinIO is accessible
+        minio_client.list_buckets()
+        return True
+    except Exception as e:
+        current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )
--- a/eveai_app/init.py
+++ b/eveai_app/init.py
@@ -7,7 +7,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix
 import logging.config

 from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session,
-                               minio_client, simple_encryption)
+                               minio_client, simple_encryption, metrics)
 from common.models.user import User, Role, Tenant, TenantDomain
 import common.models.interaction
 from common.utils.nginx_utils import prefixed_url_for
@@ -114,10 +114,10 @@ def register_extensions(app):
    csrf.init_app(app)
    login_manager.init_app(app)
    cors.init_app(app)
-    # kms_client.init_app(app)
    simple_encryption.init_app(app)
    session.init_app(app)
    minio_client.init_app(app)
+    metrics.init_app(app)


 # Register Blueprints
@@ -132,3 +132,7 @@ def register_blueprints(app):
    app.register_blueprint(security_bp)
    from .views.interaction_views import interaction_bp
    app.register_blueprint(interaction_bp)
+    from .views.healthz_views import healthz_bp, init_healtz
+    app.register_blueprint(healthz_bp)
+    init_healtz(app)
+
--- a/eveai_app/templates/macros.html
+++ b/eveai_app/templates/macros.html
@@ -1,16 +1,16 @@
-{% macro render_field(field, disabled_fields=[], exclude_fields=[]) %}
+{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %}
    {% set disabled = field.name in disabled_fields %}
    {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %}
    {% if field.name not in exclude_fields %}
        {% if field.type == 'BooleanField' %}
            <div class="form-check">
-                {{ field(class="form-check-input", type="checkbox", id="flexSwitchCheckDefault") }}
+                {{ field(class="form-check-input " + class, type="checkbox", id="flexSwitchCheckDefault") }}
                {{ field.label(class="form-check-label", for="flexSwitchCheckDefault", disabled=disabled) }}
            </div>
        {% else %}
            <div class="form-group">
                {{ field.label(class="form-label") }}
-                {{ field(class="form-control", disabled=disabled) }}
+                {{ field(class="form-control " + class, disabled=disabled) }}
                {% if field.errors %}
                    <div class="invalid-feedback">
                        {% for error in field.errors %}
--- a/eveai_app/templates/scripts.html
+++ b/eveai_app/templates/scripts.html
@@ -13,3 +13,5 @@
        <script src="{{url_for('static', filename='assets/js/plugins/anime.min.js')}}"></script>
        <script src="{{url_for('static', filename='assets/js/material-kit-pro.min.js')}}?v=3.0.4 type="text/javascript"></script>
        <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/js/bootstrap.bundle.min.js"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.13/js/select2.min.js"></script>
+
--- a/eveai_app/templates/user/select_tenant.html
+++ b/eveai_app/templates/user/select_tenant.html
@@ -1,22 +1,52 @@
 {% extends 'base.html' %}
-{% from "macros.html" import render_selectable_table, render_pagination %}
-
+{% from "macros.html" import render_selectable_table, render_pagination, render_field %}
 {% block title %}Tenant Selection{% endblock %}
-
 {% block content_title %}Select a Tenant{% endblock %}
 {% block content_description %}Select the active tenant for the current session{% endblock %}
-
 {% block content %}
+
+<!-- Filter Form -->
+<form method="POST" action="{{ url_for('user_bp.select_tenant') }}" class="mb-4">
+    {{ filter_form.hidden_tag() }}
+    <div class="row">
+        <div class="col-md-4">
+            {{ render_field(filter_form.types, class="select2") }}
+        </div>
+        <div class="col-md-4">
+            {{ render_field(filter_form.search) }}
+        </div>
+        <div class="col-md-4">
+            {{ filter_form.submit(class="btn btn-primary") }}
+        </div>
+    </div>
+</form>
+
+<!-- Tenant Selection Form -->
 <form method="POST" action="{{ url_for('user_bp.handle_tenant_selection') }}">
-    {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website"], rows=rows, selectable=True, id="tenantsTable") }}
+    {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website", "Type"], rows=rows, selectable=True, id="tenantsTable") }}
    <div class="form-group mt-3">
        <button type="submit" name="action" value="select_tenant" class="btn btn-primary">Set Session Tenant</button>
        <button type="submit" name="action" value="edit_tenant" class="btn btn-secondary">Edit Tenant</button>
    </div>
 </form>
+
 {% endblock %}

 {% block content_footer %}
-    {{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{{ render_pagination(pagination, 'user_bp.select_tenant') }}
+{% endblock %}
+
+{% block scripts %}
+<script>
+$(document).ready(function() {
+    $('.select2').select2({
+        placeholder: "Select tenant types",
+        allowClear: true,
+        minimumResultsForSearch: Infinity, // Hides the search box
+        dropdownCssClass: 'select2-dropdown-hidden', // Custom class for dropdown
+        containerCssClass: 'select2-container-hidden' // Custom class for container
+    });
+});
+</script>
 {% endblock %}

--- a/eveai_app/templates/user/tenant_overview.html
+++ b/eveai_app/templates/user/tenant_overview.html
@@ -10,7 +10,7 @@
    <form method="post">
        {{ form.hidden_tag() }}
        <!-- Main Tenant Information -->
-        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages'] %}
+        {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages', 'rag_context', 'type'] %}
        {% for field in form %}
            {{ render_included_field(field, disabled_fields=main_fields, include_fields=main_fields) }}
        {% endfor %}
--- a/eveai_app/views/document_forms.py
+++ b/eveai_app/views/document_forms.py
@@ -30,7 +30,6 @@ class AddDocumentForm(FlaskForm):
    user_context = TextAreaField('User Context', validators=[Optional()])
    valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
    user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
-    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])

    submit = SubmitField('Submit')

@@ -38,6 +37,7 @@ class AddDocumentForm(FlaskForm):
        super().__init__()
        self.language.choices = [(language, language) for language in
                                 session.get('tenant').get('allowed_languages')]
+        if not self.language.data:
            self.language.data = session.get('tenant').get('default_language')


@@ -48,7 +48,6 @@ class AddURLForm(FlaskForm):
    user_context = TextAreaField('User Context', validators=[Optional()])
    valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
    user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
-    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])

    submit = SubmitField('Submit')

@@ -56,6 +55,7 @@ class AddURLForm(FlaskForm):
        super().__init__()
        self.language.choices = [(language, language) for language in
                                 session.get('tenant').get('allowed_languages')]
+        if not self.language.data:
            self.language.data = session.get('tenant').get('default_language')


@@ -72,6 +72,7 @@ class AddURLsForm(FlaskForm):
        super().__init__()
        self.language.choices = [(language, language) for language in
                                 session.get('tenant').get('allowed_languages')]
+        if not self.language.data:
            self.language.data = session.get('tenant').get('default_language')


--- a/eveai_app/views/document_views.py
+++ b/eveai_app/views/document_views.py
@@ -56,11 +56,9 @@ def before_request():
@roles_accepted('Super User', 'Tenant Admin')
 def add_document():
    form = AddDocumentForm()
-    current_app.logger.debug('Adding document')

    if form.validate_on_submit():
        try:
-            current_app.logger.debug('Validating file type')
            tenant_id = session['tenant']['id']
            file = form.file.data
            filename = secure_filename(file.filename)
@@ -68,15 +66,15 @@ def add_document():

            validate_file_type(extension)

+            current_app.logger.debug(f'Language on form: {form.language.data}')
            api_input = {
                'name': form.name.data,
                'language': form.language.data,
                'user_context': form.user_context.data,
                'valid_from': form.valid_from.data,
                'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
-                'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None
-
            }
+            current_app.logger.debug(f'Creating document stack with input {api_input}')

            new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
            task_id = start_embedding_task(tenant_id, new_doc_vers.id)
@@ -113,7 +111,6 @@ def add_url():
                'user_context': form.user_context.data,
                'valid_from': form.valid_from.data,
                'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
-                'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None
            }

            new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id)
--- a/eveai_app/views/healthz_views.py
+++ b/eveai_app/views/healthz_views.py
@@ -0,0 +1,100 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+import time
+
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+# Define Prometheus metrics
+api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint'])
+api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        "minio": check_minio(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('tasks.ping', queue='embeddings')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+def check_minio():
+    try:
+        # List buckets to check if MinIO is accessible
+        minio_client.list_buckets()
+        return True
+    except Exception as e:
+        current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
+
+
+# Custom metrics example
+@healthz_bp.before_app_request
+def before_request():
+    request.start_time = time.time()
+    api_request_counter.labels(
+        method=request.method, endpoint=request.endpoint
+    ).inc()
+
+
+@healthz_bp.after_app_request
+def after_request(response):
+    request_duration = time.time() - request.start_time
+    api_request_latency.observe(request_duration)
+    return response
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )
--- a/eveai_app/views/user_forms.py
+++ b/eveai_app/views/user_forms.py
@@ -2,7 +2,7 @@ from flask import current_app
 from flask_wtf import FlaskForm
 from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
                     SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField)
-from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional
+from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError
 import pytz

 from common.models.user import Role
@@ -18,6 +18,8 @@ class TenantForm(FlaskForm):
    timezone = SelectField('Timezone', choices=[], validators=[DataRequired()])
    # RAG context
    rag_context = TextAreaField('RAG Context', validators=[Optional()])
+    # Tenant Type
+    type = SelectField('Tenant Type', validators=[Optional()], default='Active')
    # LLM fields
    embedding_model = SelectField('Embedding Model', choices=[], validators=[DataRequired()])
    llm_model = SelectField('Large Language Model', choices=[], validators=[DataRequired()])
@@ -65,6 +67,7 @@ class TenantForm(FlaskForm):
        # Initialize fallback algorithms
        self.fallback_algorithms.choices = \
            [(algorithm, algorithm.lower()) for algorithm in current_app.config['FALLBACK_ALGORITHMS']]
+        self.type.choices = [('', 'Select Type')] + [(t, t) for t in current_app.config['TENANT_TYPES']]


 class BaseUserForm(FlaskForm):
@@ -107,4 +110,14 @@ class TenantDomainForm(FlaskForm):
    submit = SubmitField('Add Domain')


+class TenantSelectionForm(FlaskForm):
+    types = SelectMultipleField('Tenant Types', choices=[], validators=[Optional()])
+    search = StringField('Search', validators=[Optional()])
+    submit = SubmitField('Filter')
+
+    def __init__(self, *args, **kwargs):
+        super(TenantSelectionForm, self).__init__(*args, **kwargs)
+        self.types.choices = [(t, t) for t in current_app.config['TENANT_TYPES']]
+
+

--- a/eveai_app/views/user_views.py
+++ b/eveai_app/views/user_views.py
@@ -10,7 +10,7 @@ import ast
 from common.models.user import User, Tenant, Role, TenantDomain
 from common.extensions import db, security, minio_client, simple_encryption
 from common.utils.security_utils import send_confirmation_email, send_reset_email
-from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm
+from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm, TenantSelectionForm
 from common.utils.database import Database
 from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
 from common.utils.simple_encryption import generate_api_key
@@ -245,20 +245,29 @@ def edit_user(user_id):
    return render_template('user/edit_user.html', form=form, user_id=user_id)


-@user_bp.route('/select_tenant')
+@user_bp.route('/select_tenant', methods=['GET', 'POST'])
@roles_required('Super User')
 def select_tenant():
+    filter_form = TenantSelectionForm(request.form)
    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 10, type=int)

-    query = Tenant.query.order_by(Tenant.name)  # Fetch all tenants from the database
+    query = Tenant.query

-    pagination = query.paginate(page=page, per_page=per_page)
+    if filter_form.validate_on_submit():
+        if filter_form.types.data:
+            query = query.filter(Tenant.type.in_(filter_form.types.data))
+        if filter_form.search.data:
+            search = f"%{filter_form.search.data}%"
+            query = query.filter(Tenant.name.ilike(search))
+
+    query = query.order_by(Tenant.name)
+    pagination = query.paginate(page=page, per_page=per_page, error_out=False)
    tenants = pagination.items

-    rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', '')])
+    rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', ''), ('type', '')])

-    return render_template('user/select_tenant.html', rows=rows, pagination=pagination)
+    return render_template('user/select_tenant.html', rows=rows, pagination=pagination, filter_form=filter_form)


@user_bp.route('/handle_tenant_selection', methods=['POST'])
--- a/eveai_chat/init.py
+++ b/eveai_chat/init.py
@@ -3,7 +3,7 @@ import logging.config
 from flask import Flask, jsonify
 import os

-from common.extensions import db, socketio, jwt, cors, session, simple_encryption
+from common.extensions import db, socketio, jwt, cors, session, simple_encryption, metrics
 from config.logging_config import LOGGING
 from eveai_chat.socket_handlers import chat_handler
 from common.utils.cors_utils import create_cors_after_request
@@ -32,17 +32,6 @@ def create_app(config_file=None):
    app.celery = make_celery(app.name, app.config)
    init_celery(app.celery, app)

-    # Register Blueprints
-    # register_blueprints(app)
-
-    @app.route('/ping')
-    def ping():
-        return 'pong'
-
-    @app.route('/health', methods=['GET'])
-    def health():
-        return jsonify({'status': 'ok'}), 200
-
    app.logger.info("EveAI Chat Server Started Successfully")
    app.logger.info("-------------------------------------------------------------------------------------------------")
    return app
@@ -61,8 +50,8 @@ def register_extensions(app):
                      ping_interval=app.config.get('SOCKETIO_PING_INTERVAL'),
                      )
    jwt.init_app(app)
-    # kms_client.init_app(app)
    simple_encryption.init_app(app)
+    metrics.init_app(app)

    # Cors setup
    cors.init_app(app, resources={r"/chat/*": {"origins": "*"}})
@@ -71,6 +60,7 @@ def register_extensions(app):
    session.init_app(app)


+
 def register_blueprints(app):
-    from .views.chat_views import chat_bp
-    app.register_blueprint(chat_bp)
+    from views.healthz_views import healthz_bp
+    app.register_blueprint(healthz_bp)
--- a/eveai_chat/socket_handlers/chat_handler.py
+++ b/eveai_chat/socket_handlers/chat_handler.py
@@ -1,10 +1,13 @@
 import uuid
+from functools import wraps

 from flask_jwt_extended import create_access_token, get_jwt_identity, verify_jwt_in_request, decode_token
 from flask_socketio import emit, disconnect, join_room, leave_room
 from flask import current_app, request, session
 from sqlalchemy.exc import SQLAlchemyError
 from datetime import datetime, timedelta
+from prometheus_client import Counter, Histogram
+from time import time

 from common.extensions import socketio, db, simple_encryption
 from common.models.user import Tenant
@@ -12,8 +15,27 @@ from common.models.interaction import Interaction
 from common.utils.celery_utils import current_celery
 from common.utils.database import Database

+# Define custom metrics
+socketio_message_counter = Counter('socketio_message_count', 'Count of SocketIO messages', ['event_type'])
+socketio_message_latency = Histogram('socketio_message_latency_seconds', 'Latency of SocketIO message processing', ['event_type'])
+
+
+# Decorator to measure SocketIO events
+def track_socketio_event(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        event_type = func.__name__
+        socketio_message_counter.labels(event_type=event_type).inc()
+        start_time = time()
+        result = func(*args, **kwargs)
+        latency = time() - start_time
+        socketio_message_latency.labels(event_type=event_type).observe(latency)
+        return result
+    return wrapper
+

@socketio.on('connect')
+@track_socketio_event
 def handle_connect():
    try:
        current_app.logger.debug(f'SocketIO: Connection handling started using {request.args}')
@@ -58,6 +80,7 @@ def handle_connect():


@socketio.on('disconnect')
+@track_socketio_event
 def handle_disconnect():
    room = session.get('room')
    if room:
--- a/eveai_chat/views/chat_views.py
+++ b/eveai_chat/views/chat_views.py
@@ -1,77 +0,0 @@
-from datetime import datetime as dt, timezone as tz
-from flask import request, redirect, url_for, render_template, Blueprint, session, current_app, jsonify
-from flask_security import hash_password, roles_required, roles_accepted
-from sqlalchemy.exc import SQLAlchemyError
-from flask_jwt_extended import create_access_token, jwt_required, get_jwt_identity
-from flask_socketio import emit, join_room, leave_room
-import ast
-
-
-from common.models.user import User, Tenant
-from common.models.interaction import ChatSession, Interaction, InteractionEmbedding
-from common.models.document import Embedding
-from common.extensions import db, socketio, kms_client
-from common.utils.database import Database
-
-chat_bp = Blueprint('chat_bp', __name__, url_prefix='/chat')
-
-
-@chat_bp.route('/register_client', methods=['POST'])
-def register_client():
-    tenant_id = request.json.get('tenant_id')
-    api_key = request.json.get('api_key')
-
-    # Validate tenant_id and api_key here (e.g., check against the database)
-    if validate_tenant(tenant_id, api_key):
-        access_token = create_access_token(identity={'tenant_id': tenant_id, 'api_key': api_key})
-        current_app.logger.debug(f'Tenant Registration: Tenant {tenant_id} registered successfully')
-        return jsonify({'token': access_token}), 200
-    else:
-        current_app.logger.debug(f'Tenant Registration: Invalid tenant_id ({tenant_id}) or api_key ({api_key})')
-        return jsonify({'message': 'Invalid credentials'}), 401
-
-
-@socketio.on('connect', namespace='/chat')
-@jwt_required()
-def handle_connect():
-    current_tenant = get_jwt_identity()
-    current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} connected')
-
-
-@socketio.on('message', namespace='/chat')
-@jwt_required()
-def handle_message(data):
-    current_tenant = get_jwt_identity()
-    current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} sent a message: {data}')
-    # Store interaction in the database
-    emit('response', {'data': 'Message received'}, broadcast=True)
-
-
-def validate_tenant(tenant_id, api_key):
-    tenant = Tenant.query.get_or_404(tenant_id)
-    encrypted_api_key = ast.literal_eval(tenant.encrypted_chat_api_key)
-
-    decrypted_api_key = kms_client.decrypt_api_key(encrypted_api_key)
-
-    return decrypted_api_key == api_key
-
-
-
-# @chat_bp.route('/', methods=['GET', 'POST'])
-# def chat():
-#     return render_template('chat.html')
-#
-#
-# @chat.record_once
-# def on_register(state):
-#     # TODO: write initialisation code when the blueprint is registered (only once)
-#     # socketio.init_app(state.app)
-#     pass
-#
-#
-# @socketio.on('message', namespace='/chat')
-# def handle_message(message):
-#     # TODO: write message handling code to actually realise chat
-#     # print('Received message:', message)
-#     # socketio.emit('response', {'data': message}, namespace='/chat')
-#     pass
--- a/eveai_chat/views/healthz_views.py
+++ b/eveai_chat/views/healthz_views.py
@@ -0,0 +1,70 @@
+from flask import Blueprint, current_app, request
+from flask_healthz import HealthError
+from sqlalchemy.exc import SQLAlchemyError
+from celery.exceptions import TimeoutError as CeleryTimeoutError
+from common.extensions import db, metrics, minio_client
+from common.utils.celery_utils import current_celery
+from eveai_chat.socket_handlers.chat_handler import socketio_message_counter, socketio_message_latency
+
+healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz')
+
+
+def liveness():
+    try:
+        # Basic check to see if the app is running
+        return True
+    except Exception:
+        raise HealthError("Liveness check failed")
+
+
+def readiness():
+    checks = {
+        "database": check_database(),
+        "celery": check_celery(),
+        # Add more checks as needed
+    }
+
+    if not all(checks.values()):
+        raise HealthError("Readiness check failed")
+
+
+def check_database():
+    try:
+        # Perform a simple database query
+        db.session.execute("SELECT 1")
+        return True
+    except SQLAlchemyError:
+        current_app.logger.error("Database check failed", exc_info=True)
+        return False
+
+
+def check_celery():
+    try:
+        # Send a simple task to Celery
+        result = current_celery.send_task('tasks.ping', queue='llm_interactions')
+        response = result.get(timeout=10)  # Wait for up to 10 seconds for a response
+        return response == 'pong'
+    except CeleryTimeoutError:
+        current_app.logger.error("Celery check timed out", exc_info=True)
+        return False
+    except Exception as e:
+        current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True)
+        return False
+
+
+@healthz_bp.route('/metrics')
+@metrics.do_not_track()
+def prometheus_metrics():
+    return metrics.generate_latest()
+
+
+def init_healtz(app):
+    app.config.update(
+        HEALTHZ={
+            "live": "healthz_views.liveness",
+            "ready": "healthz_views.readiness",
+        }
+    )
+    # Register SocketIO metrics with Prometheus
+    metrics.register(socketio_message_counter)
+    metrics.register(socketio_message_latency)
--- a/eveai_chat_workers/tasks.py
+++ b/eveai_chat_workers/tasks.py
@@ -26,6 +26,12 @@ from common.langchain.EveAIRetriever import EveAIRetriever
 from common.langchain.EveAIHistoryRetriever import EveAIHistoryRetriever


+# Healthcheck task
+@current_celery.task(name='ping', queue='llm_interactions')
+def ping():
+    return 'pong'
+
+
 def detail_question(question, language, model_variables, session_id):
    retriever = EveAIHistoryRetriever(model_variables, session_id)
    llm = model_variables['llm']
--- a/eveai_workers/Processors/audio_processor.py
+++ b/eveai_workers/Processors/audio_processor.py
@@ -1,27 +1,22 @@
 import io
 import os
+
 from pydub import AudioSegment
 import tempfile
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
-from common.utils.model_utils import create_language_template
-from .processor import Processor
 import subprocess

+from .transcription_processor import TranscriptionProcessor

-class AudioProcessor(Processor):
+
+class AudioProcessor(TranscriptionProcessor):
    def __init__(self, tenant, model_variables, document_version):
        super().__init__(tenant, model_variables, document_version)
        self.transcription_client = model_variables['transcription_client']
        self.transcription_model = model_variables['transcription_model']
        self.ffmpeg_path = 'ffmpeg'

-
-    def process(self):
-        self._log("Starting Audio processing")
-        try:
+    def _get_transcription(self):
        file_data = minio_client.download_document_file(
            self.tenant.id,
            self.document_version.doc_id,
@@ -29,17 +24,8 @@ class AudioProcessor(Processor):
            self.document_version.id,
            self.document_version.file_name
        )
-
        compressed_audio = self._compress_audio(file_data)
-            transcription = self._transcribe_audio(compressed_audio)
-            markdown, title = self._generate_markdown_from_transcription(transcription)
-
-            self._save_markdown(markdown)
-            self._log("Finished processing Audio")
-            return markdown, title
-        except Exception as e:
-            self._log(f"Error processing Audio: {str(e)}", level='error')
-            raise
+        return self._transcribe_audio(compressed_audio)

    def _compress_audio(self, audio_data):
        self._log("Compressing audio")
@@ -159,29 +145,3 @@ class AudioProcessor(Processor):

        return full_transcription

-    def _generate_markdown_from_transcription(self, transcription):
-        self._log("Generating markdown from transcription")
-        llm = self.model_variables['llm']
-        template = self.model_variables['transcript_template']
-        language_template = create_language_template(template, self.document_version.language)
-        transcript_prompt = ChatPromptTemplate.from_template(language_template)
-        setup = RunnablePassthrough()
-        output_parser = StrOutputParser()
-
-        chain = setup | transcript_prompt | llm | output_parser
-
-        input_transcript = {'transcript': transcription}
-        markdown = chain.invoke(input_transcript)
-
-        # Extract title from the markdown
-        title = self._extract_title_from_markdown(markdown)
-
-        return markdown, title
-
-    def _extract_title_from_markdown(self, markdown):
-        # Simple extraction of the first header as the title
-        lines = markdown.split('\n')
-        for line in lines:
-            if line.startswith('# '):
-                return line[2:].strip()
-        return "Untitled Audio Transcription"
--- a/eveai_workers/Processors/html_processor.py
+++ b/eveai_workers/Processors/html_processor.py
@@ -14,6 +14,9 @@ class HTMLProcessor(Processor):
        self.html_end_tags = model_variables['html_end_tags']
        self.html_included_elements = model_variables['html_included_elements']
        self.html_excluded_elements = model_variables['html_excluded_elements']
+        self.chunk_size = model_variables['processing_chunk_size']  # Adjust this based on your LLM's optimal input size
+        self.chunk_overlap = model_variables[
+            'processing_chunk_overlap']  # Adjust for context preservation between chunks

    def process(self):
        self._log("Starting HTML processing")
@@ -70,7 +73,7 @@ class HTMLProcessor(Processor):
        chain = setup | parse_prompt | llm | output_parser

        soup = BeautifulSoup(html_content, 'lxml')
-        chunks = self._split_content(soup)
+        chunks = self._split_content(soup, self.chunk_size)

        markdown_chunks = []
        for chunk in chunks:
--- a/eveai_workers/Processors/pdf_processor.py
+++ b/eveai_workers/Processors/pdf_processor.py
@@ -16,10 +16,10 @@ class PDFProcessor(Processor):
    def __init__(self, tenant, model_variables, document_version):
        super().__init__(tenant, model_variables, document_version)
        # PDF-specific initialization
-        self.chunk_size = model_variables['PDF_chunk_size']
-        self.chunk_overlap = model_variables['PDF_chunk_overlap']
-        self.min_chunk_size = model_variables['PDF_min_chunk_size']
-        self.max_chunk_size = model_variables['PDF_max_chunk_size']
+        self.chunk_size = model_variables['processing_chunk_size']
+        self.chunk_overlap = model_variables['processing_chunk_overlap']
+        self.min_chunk_size = model_variables['processing_min_chunk_size']
+        self.max_chunk_size = model_variables['processing_max_chunk_size']

    def process(self):
        self._log("Starting PDF processing")
@@ -228,12 +228,7 @@ class PDFProcessor(Processor):
        for chunk in chunks:
            input = {"pdf_content": chunk}
            result = chain.invoke(input)
-            # Remove Markdown code block delimiters if present
-            result = result.strip()
-            if result.startswith("```markdown"):
-                result = result[len("```markdown"):].strip()
-            if result.endswith("```"):
-                result = result[:-3].strip()
+            result = self._clean_markdown(result)
            markdown_chunks.append(result)

        return "\n\n".join(markdown_chunks)
--- a/eveai_workers/Processors/processor.py
+++ b/eveai_workers/Processors/processor.py
@@ -40,3 +40,13 @@ class Processor(ABC):
            filename,
            content.encode('utf-8')
        )
+
+    def _clean_markdown(self, markdown):
+        markdown = markdown.strip()
+        if markdown.startswith("```markdown"):
+            markdown = markdown[len("```markdown"):].strip()
+        if markdown.endswith("```"):
+            markdown = markdown[:-3].strip()
+
+        return markdown
+
--- a/eveai_workers/Processors/srt_processor.py
+++ b/eveai_workers/Processors/srt_processor.py
@@ -1,19 +1,10 @@
-import re
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnablePassthrough
 from common.extensions import minio_client
-from common.utils.model_utils import create_language_template
-from .processor import Processor
+from .transcription_processor import TranscriptionProcessor
+import re


-class SRTProcessor(Processor):
-    def __init__(self, tenant, model_variables, document_version):
-        super().__init__(tenant, model_variables, document_version)
-
-    def process(self):
-        self._log("Starting SRT processing")
-        try:
+class SRTProcessor(TranscriptionProcessor):
+    def _get_transcription(self):
        file_data = minio_client.download_document_file(
            self.tenant.id,
            self.document_version.doc_id,
@@ -21,17 +12,8 @@ class SRTProcessor(Processor):
            self.document_version.id,
            self.document_version.file_name
        )
-
        srt_content = file_data.decode('utf-8')
-            cleaned_transcription = self._clean_srt(srt_content)
-            markdown, title = self._generate_markdown_from_transcription(cleaned_transcription)
-
-            self._save_markdown(markdown)
-            self._log("Finished processing SRT")
-            return markdown, title
-        except Exception as e:
-            self._log(f"Error processing SRT: {str(e)}", level='error')
-            raise
+        return self._clean_srt(srt_content)

    def _clean_srt(self, srt_content):
        # Remove timecodes and subtitle numbers
@@ -50,31 +32,3 @@ class SRTProcessor(Processor):

        return cleaned_text

-    def _generate_markdown_from_transcription(self, transcription):
-        self._log("Generating markdown from transcription")
-        llm = self.model_variables['llm']
-        template = self.model_variables['transcript_template']
-        language_template = create_language_template(template, self.document_version.language)
-        transcript_prompt = ChatPromptTemplate.from_template(language_template)
-        setup = RunnablePassthrough()
-        output_parser = StrOutputParser()
-
-        chain = setup | transcript_prompt | llm | output_parser
-
-        input_transcript = {'transcript': transcription}
-        markdown = chain.invoke(input_transcript)
-
-        # Extract title from the markdown
-        title = self._extract_title_from_markdown(markdown)
-
-        return markdown, title
-
-    def _extract_title_from_markdown(self, markdown):
-        # Simple extraction of the first header as the title
-        lines = markdown.split('\n')
-        for line in lines:
-            if line.startswith('# '):
-                return line[2:].strip()
-        return "Untitled SRT Transcription"
-
-
--- a/eveai_workers/Processors/transcription_processor.py
+++ b/eveai_workers/Processors/transcription_processor.py
@@ -0,0 +1,90 @@
+# transcription_processor.py
+from common.utils.model_utils import create_language_template
+from .processor import Processor
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+
+
+class TranscriptionProcessor(Processor):
+    def __init__(self, tenant, model_variables, document_version):
+        super().__init__(tenant, model_variables, document_version)
+        self.chunk_size = model_variables['processing_chunk_size']
+        self.chunk_overlap = model_variables['processing_chunk_overlap']
+
+    def process(self):
+        self._log("Starting Transcription processing")
+        try:
+            transcription = self._get_transcription()
+            chunks = self._chunk_transcription(transcription)
+            markdown_chunks = self._process_chunks(chunks)
+            full_markdown = self._combine_markdown_chunks(markdown_chunks)
+            self._save_markdown(full_markdown)
+            self._log("Finished processing Transcription")
+            return full_markdown, self._extract_title_from_markdown(full_markdown)
+        except Exception as e:
+            self._log(f"Error processing Transcription: {str(e)}", level='error')
+            raise
+
+    def _get_transcription(self):
+        # This method should be implemented by child classes
+        raise NotImplementedError
+
+    def _chunk_transcription(self, transcription):
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function=len,
+            separators=["\n\n", "\n", " ", ""]
+        )
+        return text_splitter.split_text(transcription)
+
+    def _process_chunks(self, chunks):
+        self._log("Generating markdown from transcription")
+        llm = self.model_variables['llm']
+        template = self.model_variables['transcript_template']
+        language_template = create_language_template(template, self.document_version.language)
+        transcript_prompt = ChatPromptTemplate.from_template(language_template)
+        setup = RunnablePassthrough()
+        output_parser = StrOutputParser()
+
+        chain = setup | transcript_prompt | llm | output_parser
+
+        markdown_chunks = []
+        previous_part = ""
+        for i, chunk in enumerate(chunks):
+            self._log(f"Processing chunk {i + 1} of {len(chunks)}")
+            self._log(f"Previous part: {previous_part}")
+            input_transcript = {
+                'transcript': chunk,
+                'previous_part': previous_part
+            }
+            markdown = chain.invoke(input_transcript)
+            markdown = self._clean_markdown(markdown)
+            markdown_chunks.append(markdown)
+
+            # Extract the last part for the next iteration
+            lines = markdown.split('\n')
+            last_header = None
+            for line in reversed(lines):
+                if line.startswith('#'):
+                    last_header = line
+                    break
+            if last_header:
+                header_index = lines.index(last_header)
+                previous_part = '\n'.join(lines[header_index:])
+            else:
+                previous_part = lines[-1] if lines else ""
+
+        return markdown_chunks
+
+    def _combine_markdown_chunks(self, markdown_chunks):
+        return "\n\n".join(markdown_chunks)
+
+    def _extract_title_from_markdown(self, markdown):
+        lines = markdown.split('\n')
+        for line in lines:
+            if line.startswith('# '):
+                return line[2:].strip()
+        return "Untitled Transcription"
--- a/eveai_workers/tasks.py
+++ b/eveai_workers/tasks.py
@@ -25,6 +25,12 @@ from eveai_workers.Processors.pdf_processor import PDFProcessor
 from eveai_workers.Processors.srt_processor import SRTProcessor


+# Healthcheck task
+@current_celery.task(name='ping', queue='embeddings')
+def ping():
+    return 'pong'
+
+
@current_celery.task(name='create_embeddings', queue='embeddings')
 def create_embeddings(tenant_id, document_version_id):
    current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id}.')
@@ -184,14 +190,21 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks):

    chunk_total_context = (f'Filename: {document_version.file_name}\n'
                           f'User Context:\n{document_version.user_context}\n\n'
+                           f'User Metadata:\n{document_version.user_metadata}\n\n'
                           f'Title: {title}\n'
-                           f'{summary}\n'
-                           f'{document_version.system_context}\n\n')
+                           f'Summary:\n{summary}\n'
+                           f'System Context:\n{document_version.system_context}\n\n'
+                           f'System Metadata:\n{document_version.system_metadata}\n\n'
+                           )
    enriched_chunks = []
    initial_chunk = (f'Filename: {document_version.file_name}\n'
                     f'User Context:\n{document_version.user_context}\n\n'
+                     f'User Metadata:\n{document_version.user_metadata}\n\n'
                     f'Title: {title}\n'
-                     f'{chunks[0]}')
+                     f'System Context:\n{document_version.system_context}\n\n'
+                     f'System Metadata:\n{document_version.system_metadata}\n\n'
+                     f'{chunks[0]}'
+                     )

    enriched_chunks.append(initial_chunk)
    for chunk in chunks[1:]:
--- a/migrations/public/versions/083ccd8206ea_add_tenant_type.py
+++ b/migrations/public/versions/083ccd8206ea_add_tenant_type.py
@@ -0,0 +1,32 @@
+"""Add Tenant Type
+
+Revision ID: 083ccd8206ea
+Revises: ce6f5b62bbfb
+Create Date: 2024-09-12 11:30:41.958117
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '083ccd8206ea'
+down_revision = 'ce6f5b62bbfb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('type', sa.String(length=20), server_default='Active', nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tenant', schema=None) as batch_op:
+        batch_op.drop_column('type')
+
+    # ### end Alembic commands ###
--- a/nginx/static/assets/css/eveai.css
+++ b/nginx/static/assets/css/eveai.css
@@ -508,3 +508,34 @@ input[type="radio"] {
    overflow-x: auto;
 }

+/* Hide Select2's custom elements */
+.select2-container-hidden {
+    position: absolute !important;
+    left: -9999px !important;
+}
+
+.select2-dropdown-hidden {
+    display: none !important;
+}
+
+/* Ensure the original select is visible and styled */
+select.select2 {
+    display: block !important;
+    width: 100% !important;
+    height: auto !important;
+    padding: .375rem .75rem !important;
+    font-size: 1rem !important;
+    line-height: 1.5 !important;
+    color: #495057 !important;
+    background-color: #fff !important;
+    background-clip: padding-box !important;
+    border: 1px solid #ced4da !important;
+    border-radius: .25rem !important;
+    transition: border-color .15s ease-in-out,box-shadow .15s ease-in-out !important;
+}
+
+/* Style for multiple select */
+select.select2[multiple] {
+    height: auto !important;
+}
+
--- a/70
+++ b/70
@@ -1,70 +0,0 @@
-eveAI/
-│
-├── .venv/
-│
-├── common/
-│   ├── models/
-│   │   ├── __init__.py
-│   │   ├── document.py
-│   │   ├── interaction.py
-│   │   └── user.py
-│   │
-│   └── utils/
-│       ├── __init__.py
-│       └── extensions.py
-│
-├── config/
-│   ├── __init__.py
-│   ├── config.py
-│   └── logging_config.py
-│
-├── eveai_app/
-│   ├── static/
-│   ├── templates/
-│   │   ├── basic/
-│   │   ├── document/
-│   │   ├── interaction/
-│   │   ├── security/
-│   │   ├── user/
-│   │   ├── base.html
-│   │   ├── footer.html
-│   │   ├── head.html
-│   │   ├── header.html
-│   │   ├── index.html
-│   │   ├── macros.html
-│   │   ├── navbar.html
-│   │   ├── navbar_macros.html
-│   │   └── scripts.html
-│   │
-│   └── views/
-│       ├── __init__.py
-│       ├── basic_views.py
-│       ├── document_forms.py
-│       ├── document_views.py
-│       ├── errors.py
-│       ├── temp/
-│       ├── user_forms.py
-│       └── user_views.py
-│
-├── eveai_workers/
-│   ├── __init__.py
-│   ├── celery_utils.py
-│   └── tasks.py
-│
-├── instance/
-├── logs/
-│   ├── app.log
-│   ├── eveai.app.log
-│   └── eveai.workers.log
-│
-├── migrations/
-│
-├── scripts/
-│   ├── run_eveai_app.py
-│   ├── run_eveai_workers.py
-│   ├── start_eveai_app.sh
-│   ├── start_eveai_workers.sh
-│   ├── start_flower.sh
-│   └── start_logdy.sh
-│
-└── requirements.txt
--- a/repopack.config.json
+++ b/repopack.config.json
@@ -0,0 +1,19 @@
+{
+  "output": {
+    "filePath": "eveai_repo.txt",
+    "style": "xml",
+    "removeComments": false,
+    "removeEmptyLines": false,
+    "topFilesLength": 5,
+    "showLineNumbers": false
+  },
+  "include": [],
+  "ignore": {
+    "useGitignore": true,
+    "useDefaultPatterns": true,
+    "customPatterns": []
+  },
+  "security": {
+    "enableSecurityCheck": true
+  }
+}
--- a/requirements.txt
+++ b/requirements.txt
@@ -74,4 +74,6 @@ pillow~=10.4.0
 pdfplumber~=0.11.4
 PyPDF2~=3.0.1
 flask-restx~=1.3.0
+prometheus-flask-exporter~=0.23.1
+flask-healthz~=1.0.1

--- a/requirements.txt.freeze
+++ b/requirements.txt.freeze
@@ -1,175 +0,0 @@
-aiohttp==3.9.5
-aiosignal==1.3.1
-alembic==1.13.1
-amqp==5.2.0
-annotated-types==0.7.0
-anyio==4.4.0
-asn1crypto==1.5.1
-attrs==23.2.0
-Babel==2.15.0
-backoff==2.2.1
-bcrypt==4.1.3
-beautifulsoup4==4.12.3
-bidict==0.23.1
-billiard==4.2.0
-blinker==1.8.2
-cachelib==0.13.0
-cachetools==5.3.3
-celery==5.4.0
-certifi==2024.6.2
-chardet==5.2.0
-charset-normalizer==3.3.2
-click==8.1.7
-click-didyoumean==0.3.1
-click-plugins==1.1.1
-click-repl==0.3.0
-colorama==0.4.6
-cors==1.0.1
-dataclasses-json==0.6.7
-deepdiff==7.0.1
-distro==1.9.0
-dnspython==2.6.1
-dominate==2.9.1
-email_validator==2.2.0
-emoji==2.12.1
-eventlet==0.36.1
-filelock==3.15.4
-filetype==1.2.0
-Flask==3.0.3
-Flask-BabelEx==0.9.4
-Flask-Bootstrap==3.3.7.1
-Flask-Cors==4.0.1
-Flask-JWT-Extended==4.6.0
-Flask-Login==0.6.3
-flask-mailman==1.1.0
-Flask-Migrate==4.0.7
-Flask-Principal==0.4.0
-Flask-Security-Too==5.4.3
-Flask-Session==0.8.0
-Flask-SocketIO==5.3.6
-Flask-SQLAlchemy==3.1.1
-Flask-WTF==1.2.1
-flower==2.0.1
-frozenlist==1.4.1
-fsspec==2024.6.0
-future==1.0.0
-gevent==24.2.1
-gevent-websocket==0.10.1
-google==3.0.0
-google-api-core==2.19.1rc0
-google-auth==2.30.0
-google-cloud-core==2.4.1
-google-cloud-kms==2.23.0
-googleapis-common-protos==1.63.2rc0
-greenlet==3.0.3
-grpc-google-iam-v1==0.13.0
-grpcio==1.63.0
-grpcio-status==1.62.2
-gunicorn==22.0.0
-h11==0.14.0
-httpcore==1.0.5
-httpx==0.27.0
-httpx-sse==0.4.0
-huggingface-hub==0.23.4
-humanize==4.9.0
-idna==3.7
-importlib_resources==6.4.0
-itsdangerous==2.2.0
-Jinja2==3.1.4
-joblib==1.4.2
-jsonpatch==1.33
-jsonpath-python==1.0.6
-jsonpointer==3.0.0
-kombu==5.3.7
-langchain==0.2.5
-langchain-community==0.2.5
-langchain-core==0.2.9
-langchain-mistralai==0.1.8
-langchain-openai==0.1.9
-langchain-postgres==0.0.9
-langchain-text-splitters==0.2.1
-langcodes==3.4.0
-langdetect==1.0.9
-langsmith==0.1.81
-language_data==1.2.0
-lxml==5.2.2
-Mako==1.3.5
-marisa-trie==1.2.0
-MarkupSafe==2.1.5
-marshmallow==3.21.3
-msgspec==0.18.6
-multidict==6.0.5
-mypy-extensions==1.0.0
-nest-asyncio==1.6.0
-nltk==3.8.1
-numpy==2.0.0
-openai==1.35.3
-ordered-set==4.1.0
-orjson==3.10.5
-packaging==24.1
-passlib==1.7.4
-pg8000==1.31.2
-pgvector==0.2.5
-prometheus_client==0.20.0
-prompt_toolkit==3.0.47
-proto-plus==1.24.0
-protobuf==5.27.1
-psycopg==3.1.19
-psycopg-pool==3.2.2
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
-pycryptodome==3.20.0
-pydantic==2.7.4
-pydantic_core==2.19.0
-pydevd-pycharm==242.18071.12
-PyJWT==2.8.0
-pypdf==4.2.0
-PySocks==1.7.1
-python-dateutil==2.9.0.post0
-python-engineio==4.9.1
-python-iso639==2024.4.27
-python-magic==0.4.27
-python-socketio==5.11.3
-pytz==2024.1
-PyYAML==6.0.2rc1
-rapidfuzz==3.9.3
-redis==5.0.4
-regex==2024.4.28
-requests==2.32.3
-requests-file==2.1.0
-requests-toolbelt==1.0.0
-rsa==4.9
-scramp==1.4.5
-setuptools==69.5.1
-simple-websocket==1.0.0
-six==1.16.0
-sniffio==1.3.1
-soupsieve==2.5
-speaklater==1.3
-SQLAlchemy==2.0.31
-tabulate==0.9.0
-tenacity==8.4.2
-tiktoken==0.7.0
-tldextract==5.1.2
-tokenizers==0.19.1
-tornado==6.4.1
-tqdm==4.66.4
-typing-inspect==0.9.0
-typing_extensions==4.12.2
-tzdata==2024.1
-unstructured==0.14.8
-unstructured-client==0.23.7
-urllib3==2.2.2
-uWSGI==2.0.26
-vine==5.1.0
-visitor==0.1.3
-wcwidth==0.2.13
-Werkzeug==3.0.3
-wrapt==1.16.0
-wsproto==1.2.0
-WTForms==3.1.2
-wtforms-html5==0.6.1
-yarl==1.9.4
-zope.event==5.0
-zope.interface==6.3
-zxcvbn==4.4.28
--- a/requirements_orig.txt
+++ b/requirements_orig.txt
@@ -1,19 +0,0 @@
-Flask~=3.0.3
-WTForms~=3.1.2
-SQLAlchemy~=2.0.29
-alembic~=1.13.1
-Werkzeug~=3.0.2
-pgvector~=0.2.5
-gevent~=24.2.1
-celery~=5.4.0
-kombu~=5.3.7
-langchain~=0.1.17
-requests~=2.31.0
-beautifulsoup4~=4.12.3
-google~=3.0.0
-redis~=5.0.4
-itsdangerous~=2.2.0
-pydantic~=2.7.1
-chardet~=5.2.0
-langcodes~=3.4.0
-pytz~=2024.1