diff --git a/.gitignore b/.gitignore index 65b2bb1..7dd3f01 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ migrations/.DS_Store migrations/public/.DS_Store scripts/.DS_Store scripts/__pycache__/run_eveai_app.cpython-312.pyc +/eveai_repo.txt diff --git a/.repopackignore b/.repopackignore new file mode 100644 index 0000000..a2a3938 --- /dev/null +++ b/.repopackignore @@ -0,0 +1,21 @@ +# Add patterns to ignore here, one per line +# Example: +# *.log +# tmp/ +logs/ +nginx/static/assets/fonts/ +nginx/static/assets/img/ +nginx/static/assets/js/ +nginx/static/scss/ +patched_packages/ +migrations/ +*material* +*nucleo* +*package* +nginx/mime.types +*.gitignore* +.python-version +.repopackignore +repopack.config.json + + diff --git a/CHANGELOG.md b/CHANGELOG.md index e452b5e..c483b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - In case of vulnerabilities. -- + +## [1.0.7-alfa] - 2024-09-12 + +### Added +- Full Document API allowing for creation, updating and invalidation of documents. +- Metadata fields (JSON) added to DocumentVersion, allowing end-users to add structured information +- Wordpress plugin eveai_sync to synchronize Wordpress content with EveAI + +### Fixed +- Maximal deduplication of code between views and api in document_utils.py + ## [1.0.6-alfa] - 2024-09-03 ### Fixed diff --git a/common/extensions.py b/common/extensions.py index 23eab81..91d28e5 100644 --- a/common/extensions.py +++ b/common/extensions.py @@ -10,8 +10,8 @@ from flask_jwt_extended import JWTManager from flask_session import Session from flask_wtf import CSRFProtect from flask_restx import Api +from prometheus_flask_exporter import PrometheusMetrics -from .utils.nginx_utils import prefixed_url_for from .utils.simple_encryption import SimpleEncryption from .utils.minio_utils import MinioClient @@ -31,3 +31,4 @@ session = Session() api_rest = Api() simple_encryption = SimpleEncryption() minio_client = MinioClient() +metrics = PrometheusMetrics.for_app_factory() diff --git a/common/models/user.py b/common/models/user.py index 626653a..d7a28eb 100644 --- a/common/models/user.py +++ b/common/models/user.py @@ -21,6 +21,7 @@ class Tenant(db.Model): website = db.Column(db.String(255), nullable=True) timezone = db.Column(db.String(50), nullable=True, default='UTC') rag_context = db.Column(db.Text, nullable=True) + type = db.Column(db.String(20), nullable=True, server_default='Active') # language information default_language = db.Column(db.String(2), nullable=True) @@ -56,7 +57,6 @@ class Tenant(db.Model): encrypted_chat_api_key = db.Column(db.String(500), nullable=True) encrypted_api_key = db.Column(db.String(500), nullable=True) - # Tuning enablers embed_tuning = db.Column(db.Boolean, nullable=True, default=False) rag_tuning = db.Column(db.Boolean, nullable=True, default=False) @@ -75,6 +75,7 @@ class Tenant(db.Model): 'website': self.website, 'timezone': self.timezone, 'rag_context': self.rag_context, + 'type': self.type, 'default_language': self.default_language, 'allowed_languages': self.allowed_languages, 'embedding_model': self.embedding_model, diff --git a/common/utils/model_utils.py b/common/utils/model_utils.py index 0d0e507..a3b9246 100644 --- a/common/utils/model_utils.py +++ b/common/utils/model_utils.py @@ -147,10 +147,10 @@ def select_model_variables(tenant): match llm_model: case 'gpt-4o' | 'gpt-4o-mini': tool_calling_supported = True - PDF_chunk_size = 10000 - PDF_chunk_overlap = 200 - PDF_min_chunk_size = 8000 - PDF_max_chunk_size = 12000 + processing_chunk_size = 10000 + processing_chunk_overlap = 200 + processing_min_chunk_size = 8000 + processing_max_chunk_size = 12000 case _: raise Exception(f'Error setting model variables for tenant {tenant.id} ' f'error: Invalid chat model') @@ -165,18 +165,18 @@ def select_model_variables(tenant): model=llm_model_ext, temperature=model_variables['RAG_temperature']) tool_calling_supported = True - PDF_chunk_size = 10000 - PDF_chunk_overlap = 200 - PDF_min_chunk_size = 8000 - PDF_max_chunk_size = 12000 + processing_chunk_size = 10000 + processing_chunk_overlap = 200 + processing_min_chunk_size = 8000 + processing_max_chunk_size = 12000 case _: raise Exception(f'Error setting model variables for tenant {tenant.id} ' f'error: Invalid chat provider') - model_variables['PDF_chunk_size'] = PDF_chunk_size - model_variables['PDF_chunk_overlap'] = PDF_chunk_overlap - model_variables['PDF_min_chunk_size'] = PDF_min_chunk_size - model_variables['PDF_max_chunk_size'] = PDF_max_chunk_size + model_variables['processing_chunk_size'] = processing_chunk_size + model_variables['processing_chunk_overlap'] = processing_chunk_overlap + model_variables['processing_min_chunk_size'] = processing_min_chunk_size + model_variables['processing_max_chunk_size'] = processing_max_chunk_size if tool_calling_supported: model_variables['cited_answer_cls'] = CitedAnswer diff --git a/config/config.py b/config/config.py index 5031ecd..04f0771 100644 --- a/config/config.py +++ b/config/config.py @@ -139,7 +139,7 @@ class Config(object): SUPPORTED_FILE_TYPES = ['pdf', 'html', 'md', 'txt', 'mp3', 'mp4', 'ogg', 'srt'] - + TENANT_TYPES = ['Active', 'Demo', 'Inactive', 'Test'] class DevConfig(Config): diff --git a/config/gc_sa_eveai.json b/config/gc_sa_eveai.json deleted file mode 100644 index 9919cab..0000000 --- a/config/gc_sa_eveai.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "eveai-420711", - "private_key_id": "e666408e75793321a6134243628346722a71b3a6", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaGTXCWpq08YD1\nOW4z+gncOlB7T/EIiEwsZgMp6pyUrNioGfiI9YN+uVR0nsUSmFf1YyerRgX7RqD5\nRc7T/OuX8iIvmloK3g7CaFezcVrjnBKcg/QsjDAt/OO3DTk4vykDlh/Kqxx73Jdv\nFH9YSV2H7ToWqIE8CTDnqe8vQS7Bq995c9fPlues31MgndRFg3CFkH0ldfZ4aGm3\n1RnBDyC+9SPQW9e7CJgNN9PWTmOT51Zyy5IRuV5OWePMQaGLVmCo5zNc/EHZEVRu\n1hxJPHL3NNmkYDY8tye8uHgjsAkv8QuwIuUSqnqjoo1/Yg+P0+9GCpePOAJRNxJS\n0YpDFWc5AgMBAAECggEACIU4/hG+bh97BD7JriFhfDDT6bg7g+pCs/hsAlxQ42jv\nOH7pyWuHJXGf5Cwx31usZAq4fcrgYnVpnyl8odIL628y9AjdI66wMuWhZnBFGJgK\nRhHcZWjW8nlXf0lBjwwFe4edzbn1AuWT5fYZ2HWDW2mthY/e8sUwqWPcWsjdifhz\nNR7V+Ia47McKXYgEKjyEObSP1NUOW24zH0DgxS52YPMwa1FoHn6+9Pr8P3TsTSO6\nh6f8tnd81DGl1UH4F5Bj/MHsQXyAMJbu44S4+rZ4Qlk+5xPp9hfCNpxWaHLIkJCg\nYXnC8UAjjyXiqyK0U0RjJf8TS1FxUI4iPepLNqp/pQKBgQDTicZnWFXmCFTnycWp\n66P3Yx0yvlKdUdfnoD/n9NdmUA3TZUlEVfb0IOm7ZFubF/zDTH87XrRiD/NVDbr8\n6bdhA1DXzraxhbfD36Hca6K74Ba4aYJsSWWwI0hL3FDSsv8c7qAIaUF2iwuHb7Y0\nRDcvZqowtQobcQC8cHLc/bI/ZwKBgQC6fMeGaU+lP6jhp9Nb/3Gz5Z1zzCu34IOo\nlgpTNZsowRKYLtjHifrEFi3XRxPKz5thMuJFniof5U4WoMYtRXy+PbgySvBpCia2\nXty05XssnLLMvLpYU5sbQvmOTe20zaIzLohRvvmqrydYIKu62NTubNeuD1L+Zr0q\nz1P5/wUgXwKBgQCW9MrRFQi3j1qHzkVwbOglsmUzwP3TpoQclw8DyIWuTZKQOMeA\nLJh+vr4NLCDzHLsT45MoGv0+vYM4PwQhV+e1I1idqLZXGMV60iv/0A/hYpjUIPch\nr38RoxwEhsRml7XWP7OUTQiaP7+Kdv3fbo6zFOB+wbLkwk90KgrOCX0aIQKBgFeK\n7esmErJjMPdFXk3om0q09nX+mWNHLOb+EDjBiGXYRM9V5oO9PQ/BzaEqh5sEXE+D\noH7H4cR5U3AB5yYnYYi41ngdf7//eO7Rl1AADhOCN9kum1eNX9mrVhU8deMTSRo3\ntNyTBwbeFF0lcRhUY5jNVW4rWW19cz3ed/B6i8CHAoGBAJ/l5rkV74Z5hg6BWNfQ\nYAg/4PLZmjnXIy5QdnWc/PYgbhn5+iVUcL9fSofFzJM1rjFnNcs3S90MGeOmfmo4\nM1WtcQFQbsCGt6+G5uEL/nf74mKUGpOqEM/XSkZ3inweWiDk3LK3iYfXCMBFouIr\n80IlzI1yMf7MVmWn3e1zPjCA\n-----END PRIVATE KEY-----\n", - "client_email": "eveai-349@eveai-420711.iam.gserviceaccount.com", - "client_id": "109927035346319712442", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/eveai-349%40eveai-420711.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" -} diff --git a/config/prompts/openai/gpt-4o.yaml b/config/prompts/openai/gpt-4o.yaml index de9d0ec..ee1d513 100644 --- a/config/prompts/openai/gpt-4o.yaml +++ b/config/prompts/openai/gpt-4o.yaml @@ -65,11 +65,13 @@ encyclopedia: | transcript: | You are a top administrative assistant specialized in transforming given transcriptions into markdown formatted files. The generated files will be used to generate embeddings in a RAG-system. The transcriptions originate from podcast, videos and similar material. + You may receive information in different chunks. If you're not receiving the first chunk, you'll get the last part of the previous chunk, including it's title in between triple $. Consider this last part and the title as the start of the new chunk. + # Best practices and steps are: - Respect wordings and language(s) used in the transcription. Main language is {language}. - Sometimes, the transcript contains speech of several people participating in a conversation. Although these are not obvious from reading the file, try to detect when other people are speaking. - - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. + - Divide the transcript into several logical parts. Ensure questions and their answers are in the same logical part. Don't make logical parts too small. They should contain at least 7 or 8 sentences. - annotate the text to identify these logical parts using headings in {language}. - improve errors in the transcript given the context, but do not change the meaning and intentions of the transcription. @@ -77,4 +79,6 @@ transcript: | The transcript is between triple backquotes. + $$${previous_part}$$$ + ```{transcript}``` \ No newline at end of file diff --git a/docker/compose_dev.yaml b/docker/compose_dev.yaml index 33d600d..6470638 100644 --- a/docker/compose_dev.yaml +++ b/docker/compose_dev.yaml @@ -96,12 +96,11 @@ services: minio: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:5001/health"] - interval: 10s - timeout: 5s - retries: 5 -# entrypoint: ["scripts/entrypoint.sh"] -# command: ["scripts/start_eveai_app.sh"] + test: ["CMD", "curl", "-f", "http://localhost:5001/healthz/ready"] + interval: 30s + timeout: 1s + retries: 3 + start_period: 30s networks: - eveai-network @@ -113,8 +112,6 @@ services: platforms: - linux/amd64 - linux/arm64 -# ports: -# - 5001:5001 environment: <<: *common-variables COMPONENT_NAME: eveai_workers @@ -132,13 +129,6 @@ services: condition: service_healthy minio: condition: service_healthy -# healthcheck: -# test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ] -# interval: 10s -# timeout: 5s -# retries: 5 -# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ] -# command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ] networks: - eveai-network @@ -168,12 +158,11 @@ services: redis: condition: service_healthy healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ] # Adjust based on your health endpoint - interval: 10s - timeout: 5s - retries: 5 -# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ] -# command: ["sh", "-c", "scripts/start_eveai_chat.sh"] + test: [ "CMD", "curl", "-f", "http://localhost:5002/healthz/ready" ] # Adjust based on your health endpoint + interval: 30s + timeout: 1s + retries: 3 + start_period: 30s networks: - eveai-network @@ -185,8 +174,6 @@ services: platforms: - linux/amd64 - linux/arm64 -# ports: -# - 5001:5001 environment: <<: *common-variables COMPONENT_NAME: eveai_chat_workers @@ -202,13 +189,6 @@ services: condition: service_healthy redis: condition: service_healthy -# healthcheck: -# test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ] -# interval: 10s -# timeout: 5s -# retries: 5 -# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ] -# command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ] networks: - eveai-network @@ -240,12 +220,11 @@ services: minio: condition: service_healthy healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:5003/health" ] - interval: 10s - timeout: 5s - retries: 5 - # entrypoint: ["scripts/entrypoint.sh"] - # command: ["scripts/start_eveai_api.sh"] + test: [ "CMD", "curl", "-f", "http://localhost:5003/healthz/ready" ] + interval: 30s + timeout: 1s + retries: 3 + start_period: 30s networks: - eveai-network diff --git a/eveai_api/__init__.py b/eveai_api/__init__.py index 47a2a56..1214eda 100644 --- a/eveai_api/__init__.py +++ b/eveai_api/__init__.py @@ -39,9 +39,12 @@ def create_app(config_file=None): # Register Necessary Extensions register_extensions(app) - # register Blueprints + # register Namespaces register_namespaces(api_rest) + # Register Blueprints + register_blueprints(app) + # Error handler for the API @app.errorhandler(EveAIException) def handle_eveai_exception(error): @@ -102,3 +105,9 @@ def register_extensions(app): def register_namespaces(app): api_rest.add_namespace(document_ns, path='/api/v1/documents') api_rest.add_namespace(auth_ns, path='/api/v1/auth') + + +def register_blueprints(app): + from .views.healthz_views import healthz_bp + app.register_blueprint(healthz_bp) + diff --git a/eveai_api/views/healthz_views.py b/eveai_api/views/healthz_views.py new file mode 100644 index 0000000..3d25e3a --- /dev/null +++ b/eveai_api/views/healthz_views.py @@ -0,0 +1,82 @@ +from flask import Blueprint, current_app, request +from flask_healthz import HealthError +from sqlalchemy.exc import SQLAlchemyError +from celery.exceptions import TimeoutError as CeleryTimeoutError +from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST +from common.extensions import db, metrics, minio_client +from common.utils.celery_utils import current_celery + +healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz') + +# Define Prometheus metrics +api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint']) +api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency') + + +def liveness(): + try: + # Basic check to see if the app is running + return True + except Exception: + raise HealthError("Liveness check failed") + + +def readiness(): + checks = { + "database": check_database(), + "celery": check_celery(), + "minio": check_minio(), + # Add more checks as needed + } + + if not all(checks.values()): + raise HealthError("Readiness check failed") + + +def check_database(): + try: + # Perform a simple database query + db.session.execute("SELECT 1") + return True + except SQLAlchemyError: + current_app.logger.error("Database check failed", exc_info=True) + return False + + +def check_celery(): + try: + # Send a simple task to Celery + result = current_celery.send_task('tasks.ping', queue='embeddings') + response = result.get(timeout=10) # Wait for up to 10 seconds for a response + return response == 'pong' + except CeleryTimeoutError: + current_app.logger.error("Celery check timed out", exc_info=True) + return False + except Exception as e: + current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True) + return False + + +def check_minio(): + try: + # List buckets to check if MinIO is accessible + minio_client.list_buckets() + return True + except Exception as e: + current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True) + return False + + +@healthz_bp.route('/metrics') +@metrics.do_not_track() +def prometheus_metrics(): + return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST} + + +def init_healtz(app): + app.config.update( + HEALTHZ={ + "live": "healthz_views.liveness", + "ready": "healthz_views.readiness", + } + ) diff --git a/eveai_app/__init__.py b/eveai_app/__init__.py index 2704d38..35cd1c6 100644 --- a/eveai_app/__init__.py +++ b/eveai_app/__init__.py @@ -7,7 +7,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix import logging.config from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session, - minio_client, simple_encryption) + minio_client, simple_encryption, metrics) from common.models.user import User, Role, Tenant, TenantDomain import common.models.interaction from common.utils.nginx_utils import prefixed_url_for @@ -114,10 +114,10 @@ def register_extensions(app): csrf.init_app(app) login_manager.init_app(app) cors.init_app(app) - # kms_client.init_app(app) simple_encryption.init_app(app) session.init_app(app) minio_client.init_app(app) + metrics.init_app(app) # Register Blueprints @@ -132,3 +132,7 @@ def register_blueprints(app): app.register_blueprint(security_bp) from .views.interaction_views import interaction_bp app.register_blueprint(interaction_bp) + from .views.healthz_views import healthz_bp, init_healtz + app.register_blueprint(healthz_bp) + init_healtz(app) + diff --git a/eveai_app/templates/macros.html b/eveai_app/templates/macros.html index 62a0598..c45beea 100644 --- a/eveai_app/templates/macros.html +++ b/eveai_app/templates/macros.html @@ -1,16 +1,16 @@ -{% macro render_field(field, disabled_fields=[], exclude_fields=[]) %} +{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %} {% set disabled = field.name in disabled_fields %} {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %} {% if field.name not in exclude_fields %} {% if field.type == 'BooleanField' %}
- {{ field(class="form-check-input", type="checkbox", id="flexSwitchCheckDefault") }} + {{ field(class="form-check-input " + class, type="checkbox", id="flexSwitchCheckDefault") }} {{ field.label(class="form-check-label", for="flexSwitchCheckDefault", disabled=disabled) }}
{% else %}
{{ field.label(class="form-label") }} - {{ field(class="form-control", disabled=disabled) }} + {{ field(class="form-control " + class, disabled=disabled) }} {% if field.errors %}
{% for error in field.errors %} diff --git a/eveai_app/templates/scripts.html b/eveai_app/templates/scripts.html index c5fe94b..e30d72c 100644 --- a/eveai_app/templates/scripts.html +++ b/eveai_app/templates/scripts.html @@ -13,3 +13,5 @@ + + diff --git a/eveai_app/templates/user/select_tenant.html b/eveai_app/templates/user/select_tenant.html index afbcdd9..89e2ad5 100644 --- a/eveai_app/templates/user/select_tenant.html +++ b/eveai_app/templates/user/select_tenant.html @@ -1,22 +1,52 @@ {% extends 'base.html' %} -{% from "macros.html" import render_selectable_table, render_pagination %} - +{% from "macros.html" import render_selectable_table, render_pagination, render_field %} {% block title %}Tenant Selection{% endblock %} - {% block content_title %}Select a Tenant{% endblock %} {% block content_description %}Select the active tenant for the current session{% endblock %} - {% block content %} + + +
+ {{ filter_form.hidden_tag() }} +
+
+ {{ render_field(filter_form.types, class="select2") }} +
+
+ {{ render_field(filter_form.search) }} +
+
+ {{ filter_form.submit(class="btn btn-primary") }} +
+
+
+ +
- {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website"], rows=rows, selectable=True, id="tenantsTable") }} + {{ render_selectable_table(headers=["Tenant ID", "Tenant Name", "Website", "Type"], rows=rows, selectable=True, id="tenantsTable") }}
+ {% endblock %} {% block content_footer %} - {{ render_pagination(pagination, 'user_bp.select_tenant') }} +{{ render_pagination(pagination, 'user_bp.select_tenant') }} +{% endblock %} + +{% block scripts %} + {% endblock %} diff --git a/eveai_app/templates/user/tenant_overview.html b/eveai_app/templates/user/tenant_overview.html index da08a38..3128fb5 100644 --- a/eveai_app/templates/user/tenant_overview.html +++ b/eveai_app/templates/user/tenant_overview.html @@ -10,7 +10,7 @@
{{ form.hidden_tag() }} - {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages'] %} + {% set main_fields = ['name', 'website', 'default_language', 'allowed_languages', 'rag_context', 'type'] %} {% for field in form %} {{ render_included_field(field, disabled_fields=main_fields, include_fields=main_fields) }} {% endfor %} diff --git a/eveai_app/views/document_forms.py b/eveai_app/views/document_forms.py index 505a463..d734f34 100644 --- a/eveai_app/views/document_forms.py +++ b/eveai_app/views/document_forms.py @@ -30,7 +30,6 @@ class AddDocumentForm(FlaskForm): user_context = TextAreaField('User Context', validators=[Optional()]) valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) - system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) submit = SubmitField('Submit') @@ -38,7 +37,8 @@ class AddDocumentForm(FlaskForm): super().__init__() self.language.choices = [(language, language) for language in session.get('tenant').get('allowed_languages')] - self.language.data = session.get('tenant').get('default_language') + if not self.language.data: + self.language.data = session.get('tenant').get('default_language') class AddURLForm(FlaskForm): @@ -48,7 +48,6 @@ class AddURLForm(FlaskForm): user_context = TextAreaField('User Context', validators=[Optional()]) valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()]) user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json]) - system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json]) submit = SubmitField('Submit') @@ -56,7 +55,8 @@ class AddURLForm(FlaskForm): super().__init__() self.language.choices = [(language, language) for language in session.get('tenant').get('allowed_languages')] - self.language.data = session.get('tenant').get('default_language') + if not self.language.data: + self.language.data = session.get('tenant').get('default_language') class AddURLsForm(FlaskForm): @@ -72,7 +72,8 @@ class AddURLsForm(FlaskForm): super().__init__() self.language.choices = [(language, language) for language in session.get('tenant').get('allowed_languages')] - self.language.data = session.get('tenant').get('default_language') + if not self.language.data: + self.language.data = session.get('tenant').get('default_language') class EditDocumentForm(FlaskForm): diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index b6ea986..8029c10 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -56,11 +56,9 @@ def before_request(): @roles_accepted('Super User', 'Tenant Admin') def add_document(): form = AddDocumentForm() - current_app.logger.debug('Adding document') if form.validate_on_submit(): try: - current_app.logger.debug('Validating file type') tenant_id = session['tenant']['id'] file = form.file.data filename = secure_filename(file.filename) @@ -68,15 +66,15 @@ def add_document(): validate_file_type(extension) + current_app.logger.debug(f'Language on form: {form.language.data}') api_input = { 'name': form.name.data, 'language': form.language.data, 'user_context': form.user_context.data, 'valid_from': form.valid_from.data, 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None, - 'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None - } + current_app.logger.debug(f'Creating document stack with input {api_input}') new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id) task_id = start_embedding_task(tenant_id, new_doc_vers.id) @@ -113,7 +111,6 @@ def add_url(): 'user_context': form.user_context.data, 'valid_from': form.valid_from.data, 'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None, - 'system_metadata': json.loads(form.system_metadata.data) if form.system_metadata.data else None } new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id) diff --git a/eveai_app/views/healthz_views.py b/eveai_app/views/healthz_views.py new file mode 100644 index 0000000..bdb6e84 --- /dev/null +++ b/eveai_app/views/healthz_views.py @@ -0,0 +1,100 @@ +from flask import Blueprint, current_app, request +from flask_healthz import HealthError +from sqlalchemy.exc import SQLAlchemyError +from celery.exceptions import TimeoutError as CeleryTimeoutError +from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST +import time + +from common.extensions import db, metrics, minio_client +from common.utils.celery_utils import current_celery + +healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz') + +# Define Prometheus metrics +api_request_counter = Counter('api_request_count', 'API Request Count', ['method', 'endpoint']) +api_request_latency = Histogram('api_request_latency_seconds', 'API Request latency') + + +def liveness(): + try: + # Basic check to see if the app is running + return True + except Exception: + raise HealthError("Liveness check failed") + + +def readiness(): + checks = { + "database": check_database(), + "celery": check_celery(), + "minio": check_minio(), + # Add more checks as needed + } + + if not all(checks.values()): + raise HealthError("Readiness check failed") + + +def check_database(): + try: + # Perform a simple database query + db.session.execute("SELECT 1") + return True + except SQLAlchemyError: + current_app.logger.error("Database check failed", exc_info=True) + return False + + +def check_celery(): + try: + # Send a simple task to Celery + result = current_celery.send_task('tasks.ping', queue='embeddings') + response = result.get(timeout=10) # Wait for up to 10 seconds for a response + return response == 'pong' + except CeleryTimeoutError: + current_app.logger.error("Celery check timed out", exc_info=True) + return False + except Exception as e: + current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True) + return False + + +def check_minio(): + try: + # List buckets to check if MinIO is accessible + minio_client.list_buckets() + return True + except Exception as e: + current_app.logger.error(f"MinIO check failed: {str(e)}", exc_info=True) + return False + + +@healthz_bp.route('/metrics') +@metrics.do_not_track() +def prometheus_metrics(): + return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST} + + +# Custom metrics example +@healthz_bp.before_app_request +def before_request(): + request.start_time = time.time() + api_request_counter.labels( + method=request.method, endpoint=request.endpoint + ).inc() + + +@healthz_bp.after_app_request +def after_request(response): + request_duration = time.time() - request.start_time + api_request_latency.observe(request_duration) + return response + + +def init_healtz(app): + app.config.update( + HEALTHZ={ + "live": "healthz_views.liveness", + "ready": "healthz_views.readiness", + } + ) \ No newline at end of file diff --git a/eveai_app/views/user_forms.py b/eveai_app/views/user_forms.py index 4eb49b0..ce31c00 100644 --- a/eveai_app/views/user_forms.py +++ b/eveai_app/views/user_forms.py @@ -2,7 +2,7 @@ from flask import current_app from flask_wtf import FlaskForm from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField, SelectField, SelectMultipleField, FieldList, FormField, FloatField, TextAreaField) -from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional +from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError import pytz from common.models.user import Role @@ -18,6 +18,8 @@ class TenantForm(FlaskForm): timezone = SelectField('Timezone', choices=[], validators=[DataRequired()]) # RAG context rag_context = TextAreaField('RAG Context', validators=[Optional()]) + # Tenant Type + type = SelectField('Tenant Type', validators=[Optional()], default='Active') # LLM fields embedding_model = SelectField('Embedding Model', choices=[], validators=[DataRequired()]) llm_model = SelectField('Large Language Model', choices=[], validators=[DataRequired()]) @@ -65,6 +67,7 @@ class TenantForm(FlaskForm): # Initialize fallback algorithms self.fallback_algorithms.choices = \ [(algorithm, algorithm.lower()) for algorithm in current_app.config['FALLBACK_ALGORITHMS']] + self.type.choices = [('', 'Select Type')] + [(t, t) for t in current_app.config['TENANT_TYPES']] class BaseUserForm(FlaskForm): @@ -107,4 +110,14 @@ class TenantDomainForm(FlaskForm): submit = SubmitField('Add Domain') +class TenantSelectionForm(FlaskForm): + types = SelectMultipleField('Tenant Types', choices=[], validators=[Optional()]) + search = StringField('Search', validators=[Optional()]) + submit = SubmitField('Filter') + + def __init__(self, *args, **kwargs): + super(TenantSelectionForm, self).__init__(*args, **kwargs) + self.types.choices = [(t, t) for t in current_app.config['TENANT_TYPES']] + + diff --git a/eveai_app/views/user_views.py b/eveai_app/views/user_views.py index 07f316f..ea2ab61 100644 --- a/eveai_app/views/user_views.py +++ b/eveai_app/views/user_views.py @@ -10,7 +10,7 @@ import ast from common.models.user import User, Tenant, Role, TenantDomain from common.extensions import db, security, minio_client, simple_encryption from common.utils.security_utils import send_confirmation_email, send_reset_email -from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm +from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm, TenantSelectionForm from common.utils.database import Database from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed from common.utils.simple_encryption import generate_api_key @@ -245,20 +245,29 @@ def edit_user(user_id): return render_template('user/edit_user.html', form=form, user_id=user_id) -@user_bp.route('/select_tenant') +@user_bp.route('/select_tenant', methods=['GET', 'POST']) @roles_required('Super User') def select_tenant(): + filter_form = TenantSelectionForm(request.form) page = request.args.get('page', 1, type=int) per_page = request.args.get('per_page', 10, type=int) - query = Tenant.query.order_by(Tenant.name) # Fetch all tenants from the database + query = Tenant.query - pagination = query.paginate(page=page, per_page=per_page) + if filter_form.validate_on_submit(): + if filter_form.types.data: + query = query.filter(Tenant.type.in_(filter_form.types.data)) + if filter_form.search.data: + search = f"%{filter_form.search.data}%" + query = query.filter(Tenant.name.ilike(search)) + + query = query.order_by(Tenant.name) + pagination = query.paginate(page=page, per_page=per_page, error_out=False) tenants = pagination.items - rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', '')]) + rows = prepare_table_for_macro(tenants, [('id', ''), ('name', ''), ('website', ''), ('type', '')]) - return render_template('user/select_tenant.html', rows=rows, pagination=pagination) + return render_template('user/select_tenant.html', rows=rows, pagination=pagination, filter_form=filter_form) @user_bp.route('/handle_tenant_selection', methods=['POST']) diff --git a/eveai_chat/__init__.py b/eveai_chat/__init__.py index 736f7c3..07a8b76 100644 --- a/eveai_chat/__init__.py +++ b/eveai_chat/__init__.py @@ -3,7 +3,7 @@ import logging.config from flask import Flask, jsonify import os -from common.extensions import db, socketio, jwt, cors, session, simple_encryption +from common.extensions import db, socketio, jwt, cors, session, simple_encryption, metrics from config.logging_config import LOGGING from eveai_chat.socket_handlers import chat_handler from common.utils.cors_utils import create_cors_after_request @@ -32,17 +32,6 @@ def create_app(config_file=None): app.celery = make_celery(app.name, app.config) init_celery(app.celery, app) - # Register Blueprints - # register_blueprints(app) - - @app.route('/ping') - def ping(): - return 'pong' - - @app.route('/health', methods=['GET']) - def health(): - return jsonify({'status': 'ok'}), 200 - app.logger.info("EveAI Chat Server Started Successfully") app.logger.info("-------------------------------------------------------------------------------------------------") return app @@ -61,8 +50,8 @@ def register_extensions(app): ping_interval=app.config.get('SOCKETIO_PING_INTERVAL'), ) jwt.init_app(app) - # kms_client.init_app(app) simple_encryption.init_app(app) + metrics.init_app(app) # Cors setup cors.init_app(app, resources={r"/chat/*": {"origins": "*"}}) @@ -71,6 +60,7 @@ def register_extensions(app): session.init_app(app) + def register_blueprints(app): - from .views.chat_views import chat_bp - app.register_blueprint(chat_bp) + from views.healthz_views import healthz_bp + app.register_blueprint(healthz_bp) diff --git a/eveai_chat/socket_handlers/chat_handler.py b/eveai_chat/socket_handlers/chat_handler.py index 31c0e52..b4200ec 100644 --- a/eveai_chat/socket_handlers/chat_handler.py +++ b/eveai_chat/socket_handlers/chat_handler.py @@ -1,10 +1,13 @@ import uuid +from functools import wraps from flask_jwt_extended import create_access_token, get_jwt_identity, verify_jwt_in_request, decode_token from flask_socketio import emit, disconnect, join_room, leave_room from flask import current_app, request, session from sqlalchemy.exc import SQLAlchemyError from datetime import datetime, timedelta +from prometheus_client import Counter, Histogram +from time import time from common.extensions import socketio, db, simple_encryption from common.models.user import Tenant @@ -12,8 +15,27 @@ from common.models.interaction import Interaction from common.utils.celery_utils import current_celery from common.utils.database import Database +# Define custom metrics +socketio_message_counter = Counter('socketio_message_count', 'Count of SocketIO messages', ['event_type']) +socketio_message_latency = Histogram('socketio_message_latency_seconds', 'Latency of SocketIO message processing', ['event_type']) + + +# Decorator to measure SocketIO events +def track_socketio_event(func): + @wraps(func) + def wrapper(*args, **kwargs): + event_type = func.__name__ + socketio_message_counter.labels(event_type=event_type).inc() + start_time = time() + result = func(*args, **kwargs) + latency = time() - start_time + socketio_message_latency.labels(event_type=event_type).observe(latency) + return result + return wrapper + @socketio.on('connect') +@track_socketio_event def handle_connect(): try: current_app.logger.debug(f'SocketIO: Connection handling started using {request.args}') @@ -58,6 +80,7 @@ def handle_connect(): @socketio.on('disconnect') +@track_socketio_event def handle_disconnect(): room = session.get('room') if room: diff --git a/eveai_chat/views/chat_views.py b/eveai_chat/views/chat_views.py deleted file mode 100644 index b587871..0000000 --- a/eveai_chat/views/chat_views.py +++ /dev/null @@ -1,77 +0,0 @@ -from datetime import datetime as dt, timezone as tz -from flask import request, redirect, url_for, render_template, Blueprint, session, current_app, jsonify -from flask_security import hash_password, roles_required, roles_accepted -from sqlalchemy.exc import SQLAlchemyError -from flask_jwt_extended import create_access_token, jwt_required, get_jwt_identity -from flask_socketio import emit, join_room, leave_room -import ast - - -from common.models.user import User, Tenant -from common.models.interaction import ChatSession, Interaction, InteractionEmbedding -from common.models.document import Embedding -from common.extensions import db, socketio, kms_client -from common.utils.database import Database - -chat_bp = Blueprint('chat_bp', __name__, url_prefix='/chat') - - -@chat_bp.route('/register_client', methods=['POST']) -def register_client(): - tenant_id = request.json.get('tenant_id') - api_key = request.json.get('api_key') - - # Validate tenant_id and api_key here (e.g., check against the database) - if validate_tenant(tenant_id, api_key): - access_token = create_access_token(identity={'tenant_id': tenant_id, 'api_key': api_key}) - current_app.logger.debug(f'Tenant Registration: Tenant {tenant_id} registered successfully') - return jsonify({'token': access_token}), 200 - else: - current_app.logger.debug(f'Tenant Registration: Invalid tenant_id ({tenant_id}) or api_key ({api_key})') - return jsonify({'message': 'Invalid credentials'}), 401 - - -@socketio.on('connect', namespace='/chat') -@jwt_required() -def handle_connect(): - current_tenant = get_jwt_identity() - current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} connected') - - -@socketio.on('message', namespace='/chat') -@jwt_required() -def handle_message(data): - current_tenant = get_jwt_identity() - current_app.logger.debug(f'Tenant {current_tenant["tenant_id"]} sent a message: {data}') - # Store interaction in the database - emit('response', {'data': 'Message received'}, broadcast=True) - - -def validate_tenant(tenant_id, api_key): - tenant = Tenant.query.get_or_404(tenant_id) - encrypted_api_key = ast.literal_eval(tenant.encrypted_chat_api_key) - - decrypted_api_key = kms_client.decrypt_api_key(encrypted_api_key) - - return decrypted_api_key == api_key - - - -# @chat_bp.route('/', methods=['GET', 'POST']) -# def chat(): -# return render_template('chat.html') -# -# -# @chat.record_once -# def on_register(state): -# # TODO: write initialisation code when the blueprint is registered (only once) -# # socketio.init_app(state.app) -# pass -# -# -# @socketio.on('message', namespace='/chat') -# def handle_message(message): -# # TODO: write message handling code to actually realise chat -# # print('Received message:', message) -# # socketio.emit('response', {'data': message}, namespace='/chat') -# pass diff --git a/eveai_chat/views/healthz_views.py b/eveai_chat/views/healthz_views.py new file mode 100644 index 0000000..bf4f969 --- /dev/null +++ b/eveai_chat/views/healthz_views.py @@ -0,0 +1,70 @@ +from flask import Blueprint, current_app, request +from flask_healthz import HealthError +from sqlalchemy.exc import SQLAlchemyError +from celery.exceptions import TimeoutError as CeleryTimeoutError +from common.extensions import db, metrics, minio_client +from common.utils.celery_utils import current_celery +from eveai_chat.socket_handlers.chat_handler import socketio_message_counter, socketio_message_latency + +healthz_bp = Blueprint('healthz', __name__, url_prefix='/_healthz') + + +def liveness(): + try: + # Basic check to see if the app is running + return True + except Exception: + raise HealthError("Liveness check failed") + + +def readiness(): + checks = { + "database": check_database(), + "celery": check_celery(), + # Add more checks as needed + } + + if not all(checks.values()): + raise HealthError("Readiness check failed") + + +def check_database(): + try: + # Perform a simple database query + db.session.execute("SELECT 1") + return True + except SQLAlchemyError: + current_app.logger.error("Database check failed", exc_info=True) + return False + + +def check_celery(): + try: + # Send a simple task to Celery + result = current_celery.send_task('tasks.ping', queue='llm_interactions') + response = result.get(timeout=10) # Wait for up to 10 seconds for a response + return response == 'pong' + except CeleryTimeoutError: + current_app.logger.error("Celery check timed out", exc_info=True) + return False + except Exception as e: + current_app.logger.error(f"Celery check failed: {str(e)}", exc_info=True) + return False + + +@healthz_bp.route('/metrics') +@metrics.do_not_track() +def prometheus_metrics(): + return metrics.generate_latest() + + +def init_healtz(app): + app.config.update( + HEALTHZ={ + "live": "healthz_views.liveness", + "ready": "healthz_views.readiness", + } + ) + # Register SocketIO metrics with Prometheus + metrics.register(socketio_message_counter) + metrics.register(socketio_message_latency) \ No newline at end of file diff --git a/eveai_chat_workers/tasks.py b/eveai_chat_workers/tasks.py index 3e5b84d..722a5c1 100644 --- a/eveai_chat_workers/tasks.py +++ b/eveai_chat_workers/tasks.py @@ -26,6 +26,12 @@ from common.langchain.EveAIRetriever import EveAIRetriever from common.langchain.EveAIHistoryRetriever import EveAIHistoryRetriever +# Healthcheck task +@current_celery.task(name='ping', queue='llm_interactions') +def ping(): + return 'pong' + + def detail_question(question, language, model_variables, session_id): retriever = EveAIHistoryRetriever(model_variables, session_id) llm = model_variables['llm'] diff --git a/eveai_workers/Processors/audio_processor.py b/eveai_workers/Processors/audio_processor.py index 61813b8..d3c7e0d 100644 --- a/eveai_workers/Processors/audio_processor.py +++ b/eveai_workers/Processors/audio_processor.py @@ -1,45 +1,31 @@ import io import os + from pydub import AudioSegment import tempfile -from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnablePassthrough from common.extensions import minio_client -from common.utils.model_utils import create_language_template -from .processor import Processor import subprocess +from .transcription_processor import TranscriptionProcessor -class AudioProcessor(Processor): + +class AudioProcessor(TranscriptionProcessor): def __init__(self, tenant, model_variables, document_version): super().__init__(tenant, model_variables, document_version) self.transcription_client = model_variables['transcription_client'] self.transcription_model = model_variables['transcription_model'] self.ffmpeg_path = 'ffmpeg' - - def process(self): - self._log("Starting Audio processing") - try: - file_data = minio_client.download_document_file( - self.tenant.id, - self.document_version.doc_id, - self.document_version.language, - self.document_version.id, - self.document_version.file_name - ) - - compressed_audio = self._compress_audio(file_data) - transcription = self._transcribe_audio(compressed_audio) - markdown, title = self._generate_markdown_from_transcription(transcription) - - self._save_markdown(markdown) - self._log("Finished processing Audio") - return markdown, title - except Exception as e: - self._log(f"Error processing Audio: {str(e)}", level='error') - raise + def _get_transcription(self): + file_data = minio_client.download_document_file( + self.tenant.id, + self.document_version.doc_id, + self.document_version.language, + self.document_version.id, + self.document_version.file_name + ) + compressed_audio = self._compress_audio(file_data) + return self._transcribe_audio(compressed_audio) def _compress_audio(self, audio_data): self._log("Compressing audio") @@ -159,29 +145,3 @@ class AudioProcessor(Processor): return full_transcription - def _generate_markdown_from_transcription(self, transcription): - self._log("Generating markdown from transcription") - llm = self.model_variables['llm'] - template = self.model_variables['transcript_template'] - language_template = create_language_template(template, self.document_version.language) - transcript_prompt = ChatPromptTemplate.from_template(language_template) - setup = RunnablePassthrough() - output_parser = StrOutputParser() - - chain = setup | transcript_prompt | llm | output_parser - - input_transcript = {'transcript': transcription} - markdown = chain.invoke(input_transcript) - - # Extract title from the markdown - title = self._extract_title_from_markdown(markdown) - - return markdown, title - - def _extract_title_from_markdown(self, markdown): - # Simple extraction of the first header as the title - lines = markdown.split('\n') - for line in lines: - if line.startswith('# '): - return line[2:].strip() - return "Untitled Audio Transcription" diff --git a/eveai_workers/Processors/html_processor.py b/eveai_workers/Processors/html_processor.py index 22f6aef..fbb7082 100644 --- a/eveai_workers/Processors/html_processor.py +++ b/eveai_workers/Processors/html_processor.py @@ -14,6 +14,9 @@ class HTMLProcessor(Processor): self.html_end_tags = model_variables['html_end_tags'] self.html_included_elements = model_variables['html_included_elements'] self.html_excluded_elements = model_variables['html_excluded_elements'] + self.chunk_size = model_variables['processing_chunk_size'] # Adjust this based on your LLM's optimal input size + self.chunk_overlap = model_variables[ + 'processing_chunk_overlap'] # Adjust for context preservation between chunks def process(self): self._log("Starting HTML processing") @@ -70,7 +73,7 @@ class HTMLProcessor(Processor): chain = setup | parse_prompt | llm | output_parser soup = BeautifulSoup(html_content, 'lxml') - chunks = self._split_content(soup) + chunks = self._split_content(soup, self.chunk_size) markdown_chunks = [] for chunk in chunks: diff --git a/eveai_workers/Processors/pdf_processor.py b/eveai_workers/Processors/pdf_processor.py index 7c65085..cc2e156 100644 --- a/eveai_workers/Processors/pdf_processor.py +++ b/eveai_workers/Processors/pdf_processor.py @@ -16,10 +16,10 @@ class PDFProcessor(Processor): def __init__(self, tenant, model_variables, document_version): super().__init__(tenant, model_variables, document_version) # PDF-specific initialization - self.chunk_size = model_variables['PDF_chunk_size'] - self.chunk_overlap = model_variables['PDF_chunk_overlap'] - self.min_chunk_size = model_variables['PDF_min_chunk_size'] - self.max_chunk_size = model_variables['PDF_max_chunk_size'] + self.chunk_size = model_variables['processing_chunk_size'] + self.chunk_overlap = model_variables['processing_chunk_overlap'] + self.min_chunk_size = model_variables['processing_min_chunk_size'] + self.max_chunk_size = model_variables['processing_max_chunk_size'] def process(self): self._log("Starting PDF processing") @@ -228,12 +228,7 @@ class PDFProcessor(Processor): for chunk in chunks: input = {"pdf_content": chunk} result = chain.invoke(input) - # Remove Markdown code block delimiters if present - result = result.strip() - if result.startswith("```markdown"): - result = result[len("```markdown"):].strip() - if result.endswith("```"): - result = result[:-3].strip() + result = self._clean_markdown(result) markdown_chunks.append(result) return "\n\n".join(markdown_chunks) diff --git a/eveai_workers/Processors/processor.py b/eveai_workers/Processors/processor.py index 207fd7b..361777a 100644 --- a/eveai_workers/Processors/processor.py +++ b/eveai_workers/Processors/processor.py @@ -40,3 +40,13 @@ class Processor(ABC): filename, content.encode('utf-8') ) + + def _clean_markdown(self, markdown): + markdown = markdown.strip() + if markdown.startswith("```markdown"): + markdown = markdown[len("```markdown"):].strip() + if markdown.endswith("```"): + markdown = markdown[:-3].strip() + + return markdown + diff --git a/eveai_workers/Processors/srt_processor.py b/eveai_workers/Processors/srt_processor.py index 41085d0..ccf2c6e 100644 --- a/eveai_workers/Processors/srt_processor.py +++ b/eveai_workers/Processors/srt_processor.py @@ -1,37 +1,19 @@ -import re -from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnablePassthrough from common.extensions import minio_client -from common.utils.model_utils import create_language_template -from .processor import Processor +from .transcription_processor import TranscriptionProcessor +import re -class SRTProcessor(Processor): - def __init__(self, tenant, model_variables, document_version): - super().__init__(tenant, model_variables, document_version) - - def process(self): - self._log("Starting SRT processing") - try: - file_data = minio_client.download_document_file( - self.tenant.id, - self.document_version.doc_id, - self.document_version.language, - self.document_version.id, - self.document_version.file_name - ) - - srt_content = file_data.decode('utf-8') - cleaned_transcription = self._clean_srt(srt_content) - markdown, title = self._generate_markdown_from_transcription(cleaned_transcription) - - self._save_markdown(markdown) - self._log("Finished processing SRT") - return markdown, title - except Exception as e: - self._log(f"Error processing SRT: {str(e)}", level='error') - raise +class SRTProcessor(TranscriptionProcessor): + def _get_transcription(self): + file_data = minio_client.download_document_file( + self.tenant.id, + self.document_version.doc_id, + self.document_version.language, + self.document_version.id, + self.document_version.file_name + ) + srt_content = file_data.decode('utf-8') + return self._clean_srt(srt_content) def _clean_srt(self, srt_content): # Remove timecodes and subtitle numbers @@ -50,31 +32,3 @@ class SRTProcessor(Processor): return cleaned_text - def _generate_markdown_from_transcription(self, transcription): - self._log("Generating markdown from transcription") - llm = self.model_variables['llm'] - template = self.model_variables['transcript_template'] - language_template = create_language_template(template, self.document_version.language) - transcript_prompt = ChatPromptTemplate.from_template(language_template) - setup = RunnablePassthrough() - output_parser = StrOutputParser() - - chain = setup | transcript_prompt | llm | output_parser - - input_transcript = {'transcript': transcription} - markdown = chain.invoke(input_transcript) - - # Extract title from the markdown - title = self._extract_title_from_markdown(markdown) - - return markdown, title - - def _extract_title_from_markdown(self, markdown): - # Simple extraction of the first header as the title - lines = markdown.split('\n') - for line in lines: - if line.startswith('# '): - return line[2:].strip() - return "Untitled SRT Transcription" - - diff --git a/eveai_workers/Processors/transcription_processor.py b/eveai_workers/Processors/transcription_processor.py new file mode 100644 index 0000000..09e3544 --- /dev/null +++ b/eveai_workers/Processors/transcription_processor.py @@ -0,0 +1,90 @@ +# transcription_processor.py +from common.utils.model_utils import create_language_template +from .processor import Processor +from langchain_text_splitters import RecursiveCharacterTextSplitter +from langchain_core.output_parsers import StrOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnablePassthrough + + +class TranscriptionProcessor(Processor): + def __init__(self, tenant, model_variables, document_version): + super().__init__(tenant, model_variables, document_version) + self.chunk_size = model_variables['processing_chunk_size'] + self.chunk_overlap = model_variables['processing_chunk_overlap'] + + def process(self): + self._log("Starting Transcription processing") + try: + transcription = self._get_transcription() + chunks = self._chunk_transcription(transcription) + markdown_chunks = self._process_chunks(chunks) + full_markdown = self._combine_markdown_chunks(markdown_chunks) + self._save_markdown(full_markdown) + self._log("Finished processing Transcription") + return full_markdown, self._extract_title_from_markdown(full_markdown) + except Exception as e: + self._log(f"Error processing Transcription: {str(e)}", level='error') + raise + + def _get_transcription(self): + # This method should be implemented by child classes + raise NotImplementedError + + def _chunk_transcription(self, transcription): + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=self.chunk_size, + chunk_overlap=self.chunk_overlap, + length_function=len, + separators=["\n\n", "\n", " ", ""] + ) + return text_splitter.split_text(transcription) + + def _process_chunks(self, chunks): + self._log("Generating markdown from transcription") + llm = self.model_variables['llm'] + template = self.model_variables['transcript_template'] + language_template = create_language_template(template, self.document_version.language) + transcript_prompt = ChatPromptTemplate.from_template(language_template) + setup = RunnablePassthrough() + output_parser = StrOutputParser() + + chain = setup | transcript_prompt | llm | output_parser + + markdown_chunks = [] + previous_part = "" + for i, chunk in enumerate(chunks): + self._log(f"Processing chunk {i + 1} of {len(chunks)}") + self._log(f"Previous part: {previous_part}") + input_transcript = { + 'transcript': chunk, + 'previous_part': previous_part + } + markdown = chain.invoke(input_transcript) + markdown = self._clean_markdown(markdown) + markdown_chunks.append(markdown) + + # Extract the last part for the next iteration + lines = markdown.split('\n') + last_header = None + for line in reversed(lines): + if line.startswith('#'): + last_header = line + break + if last_header: + header_index = lines.index(last_header) + previous_part = '\n'.join(lines[header_index:]) + else: + previous_part = lines[-1] if lines else "" + + return markdown_chunks + + def _combine_markdown_chunks(self, markdown_chunks): + return "\n\n".join(markdown_chunks) + + def _extract_title_from_markdown(self, markdown): + lines = markdown.split('\n') + for line in lines: + if line.startswith('# '): + return line[2:].strip() + return "Untitled Transcription" \ No newline at end of file diff --git a/eveai_workers/tasks.py b/eveai_workers/tasks.py index 8f56544..8220f8f 100644 --- a/eveai_workers/tasks.py +++ b/eveai_workers/tasks.py @@ -25,6 +25,12 @@ from eveai_workers.Processors.pdf_processor import PDFProcessor from eveai_workers.Processors.srt_processor import SRTProcessor +# Healthcheck task +@current_celery.task(name='ping', queue='embeddings') +def ping(): + return 'pong' + + @current_celery.task(name='create_embeddings', queue='embeddings') def create_embeddings(tenant_id, document_version_id): current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id}.') @@ -184,14 +190,21 @@ def enrich_chunks(tenant, model_variables, document_version, title, chunks): chunk_total_context = (f'Filename: {document_version.file_name}\n' f'User Context:\n{document_version.user_context}\n\n' + f'User Metadata:\n{document_version.user_metadata}\n\n' f'Title: {title}\n' - f'{summary}\n' - f'{document_version.system_context}\n\n') + f'Summary:\n{summary}\n' + f'System Context:\n{document_version.system_context}\n\n' + f'System Metadata:\n{document_version.system_metadata}\n\n' + ) enriched_chunks = [] initial_chunk = (f'Filename: {document_version.file_name}\n' f'User Context:\n{document_version.user_context}\n\n' + f'User Metadata:\n{document_version.user_metadata}\n\n' f'Title: {title}\n' - f'{chunks[0]}') + f'System Context:\n{document_version.system_context}\n\n' + f'System Metadata:\n{document_version.system_metadata}\n\n' + f'{chunks[0]}' + ) enriched_chunks.append(initial_chunk) for chunk in chunks[1:]: diff --git a/migrations/public/versions/083ccd8206ea_add_tenant_type.py b/migrations/public/versions/083ccd8206ea_add_tenant_type.py new file mode 100644 index 0000000..29ceed0 --- /dev/null +++ b/migrations/public/versions/083ccd8206ea_add_tenant_type.py @@ -0,0 +1,32 @@ +"""Add Tenant Type + +Revision ID: 083ccd8206ea +Revises: ce6f5b62bbfb +Create Date: 2024-09-12 11:30:41.958117 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '083ccd8206ea' +down_revision = 'ce6f5b62bbfb' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('tenant', schema=None) as batch_op: + batch_op.add_column(sa.Column('type', sa.String(length=20), server_default='Active', nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('tenant', schema=None) as batch_op: + batch_op.drop_column('type') + + # ### end Alembic commands ### diff --git a/nginx/static/assets/css/eveai.css b/nginx/static/assets/css/eveai.css index 2310b69..ababa02 100644 --- a/nginx/static/assets/css/eveai.css +++ b/nginx/static/assets/css/eveai.css @@ -508,3 +508,34 @@ input[type="radio"] { overflow-x: auto; } +/* Hide Select2's custom elements */ +.select2-container-hidden { + position: absolute !important; + left: -9999px !important; +} + +.select2-dropdown-hidden { + display: none !important; +} + +/* Ensure the original select is visible and styled */ +select.select2 { + display: block !important; + width: 100% !important; + height: auto !important; + padding: .375rem .75rem !important; + font-size: 1rem !important; + line-height: 1.5 !important; + color: #495057 !important; + background-color: #fff !important; + background-clip: padding-box !important; + border: 1px solid #ced4da !important; + border-radius: .25rem !important; + transition: border-color .15s ease-in-out,box-shadow .15s ease-in-out !important; +} + +/* Style for multiple select */ +select.select2[multiple] { + height: auto !important; +} + diff --git a/project structure b/project structure deleted file mode 100644 index 746abfc..0000000 --- a/project structure +++ /dev/null @@ -1,70 +0,0 @@ -eveAI/ -│ -├── .venv/ -│ -├── common/ -│ ├── models/ -│ │ ├── __init__.py -│ │ ├── document.py -│ │ ├── interaction.py -│ │ └── user.py -│ │ -│ └── utils/ -│ ├── __init__.py -│ └── extensions.py -│ -├── config/ -│ ├── __init__.py -│ ├── config.py -│ └── logging_config.py -│ -├── eveai_app/ -│ ├── static/ -│ ├── templates/ -│ │ ├── basic/ -│ │ ├── document/ -│ │ ├── interaction/ -│ │ ├── security/ -│ │ ├── user/ -│ │ ├── base.html -│ │ ├── footer.html -│ │ ├── head.html -│ │ ├── header.html -│ │ ├── index.html -│ │ ├── macros.html -│ │ ├── navbar.html -│ │ ├── navbar_macros.html -│ │ └── scripts.html -│ │ -│ └── views/ -│ ├── __init__.py -│ ├── basic_views.py -│ ├── document_forms.py -│ ├── document_views.py -│ ├── errors.py -│ ├── temp/ -│ ├── user_forms.py -│ └── user_views.py -│ -├── eveai_workers/ -│ ├── __init__.py -│ ├── celery_utils.py -│ └── tasks.py -│ -├── instance/ -├── logs/ -│ ├── app.log -│ ├── eveai.app.log -│ └── eveai.workers.log -│ -├── migrations/ -│ -├── scripts/ -│ ├── run_eveai_app.py -│ ├── run_eveai_workers.py -│ ├── start_eveai_app.sh -│ ├── start_eveai_workers.sh -│ ├── start_flower.sh -│ └── start_logdy.sh -│ -└── requirements.txt diff --git a/repopack.config.json b/repopack.config.json new file mode 100644 index 0000000..d324ff4 --- /dev/null +++ b/repopack.config.json @@ -0,0 +1,19 @@ +{ + "output": { + "filePath": "eveai_repo.txt", + "style": "xml", + "removeComments": false, + "removeEmptyLines": false, + "topFilesLength": 5, + "showLineNumbers": false + }, + "include": [], + "ignore": { + "useGitignore": true, + "useDefaultPatterns": true, + "customPatterns": [] + }, + "security": { + "enableSecurityCheck": true + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index fa1e571..e7322cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -74,4 +74,6 @@ pillow~=10.4.0 pdfplumber~=0.11.4 PyPDF2~=3.0.1 flask-restx~=1.3.0 +prometheus-flask-exporter~=0.23.1 +flask-healthz~=1.0.1 diff --git a/requirements.txt.freeze b/requirements.txt.freeze deleted file mode 100644 index c7e2030..0000000 --- a/requirements.txt.freeze +++ /dev/null @@ -1,175 +0,0 @@ -aiohttp==3.9.5 -aiosignal==1.3.1 -alembic==1.13.1 -amqp==5.2.0 -annotated-types==0.7.0 -anyio==4.4.0 -asn1crypto==1.5.1 -attrs==23.2.0 -Babel==2.15.0 -backoff==2.2.1 -bcrypt==4.1.3 -beautifulsoup4==4.12.3 -bidict==0.23.1 -billiard==4.2.0 -blinker==1.8.2 -cachelib==0.13.0 -cachetools==5.3.3 -celery==5.4.0 -certifi==2024.6.2 -chardet==5.2.0 -charset-normalizer==3.3.2 -click==8.1.7 -click-didyoumean==0.3.1 -click-plugins==1.1.1 -click-repl==0.3.0 -colorama==0.4.6 -cors==1.0.1 -dataclasses-json==0.6.7 -deepdiff==7.0.1 -distro==1.9.0 -dnspython==2.6.1 -dominate==2.9.1 -email_validator==2.2.0 -emoji==2.12.1 -eventlet==0.36.1 -filelock==3.15.4 -filetype==1.2.0 -Flask==3.0.3 -Flask-BabelEx==0.9.4 -Flask-Bootstrap==3.3.7.1 -Flask-Cors==4.0.1 -Flask-JWT-Extended==4.6.0 -Flask-Login==0.6.3 -flask-mailman==1.1.0 -Flask-Migrate==4.0.7 -Flask-Principal==0.4.0 -Flask-Security-Too==5.4.3 -Flask-Session==0.8.0 -Flask-SocketIO==5.3.6 -Flask-SQLAlchemy==3.1.1 -Flask-WTF==1.2.1 -flower==2.0.1 -frozenlist==1.4.1 -fsspec==2024.6.0 -future==1.0.0 -gevent==24.2.1 -gevent-websocket==0.10.1 -google==3.0.0 -google-api-core==2.19.1rc0 -google-auth==2.30.0 -google-cloud-core==2.4.1 -google-cloud-kms==2.23.0 -googleapis-common-protos==1.63.2rc0 -greenlet==3.0.3 -grpc-google-iam-v1==0.13.0 -grpcio==1.63.0 -grpcio-status==1.62.2 -gunicorn==22.0.0 -h11==0.14.0 -httpcore==1.0.5 -httpx==0.27.0 -httpx-sse==0.4.0 -huggingface-hub==0.23.4 -humanize==4.9.0 -idna==3.7 -importlib_resources==6.4.0 -itsdangerous==2.2.0 -Jinja2==3.1.4 -joblib==1.4.2 -jsonpatch==1.33 -jsonpath-python==1.0.6 -jsonpointer==3.0.0 -kombu==5.3.7 -langchain==0.2.5 -langchain-community==0.2.5 -langchain-core==0.2.9 -langchain-mistralai==0.1.8 -langchain-openai==0.1.9 -langchain-postgres==0.0.9 -langchain-text-splitters==0.2.1 -langcodes==3.4.0 -langdetect==1.0.9 -langsmith==0.1.81 -language_data==1.2.0 -lxml==5.2.2 -Mako==1.3.5 -marisa-trie==1.2.0 -MarkupSafe==2.1.5 -marshmallow==3.21.3 -msgspec==0.18.6 -multidict==6.0.5 -mypy-extensions==1.0.0 -nest-asyncio==1.6.0 -nltk==3.8.1 -numpy==2.0.0 -openai==1.35.3 -ordered-set==4.1.0 -orjson==3.10.5 -packaging==24.1 -passlib==1.7.4 -pg8000==1.31.2 -pgvector==0.2.5 -prometheus_client==0.20.0 -prompt_toolkit==3.0.47 -proto-plus==1.24.0 -protobuf==5.27.1 -psycopg==3.1.19 -psycopg-pool==3.2.2 -pyasn1==0.6.0 -pyasn1_modules==0.4.0 -pycryptodome==3.20.0 -pydantic==2.7.4 -pydantic_core==2.19.0 -pydevd-pycharm==242.18071.12 -PyJWT==2.8.0 -pypdf==4.2.0 -PySocks==1.7.1 -python-dateutil==2.9.0.post0 -python-engineio==4.9.1 -python-iso639==2024.4.27 -python-magic==0.4.27 -python-socketio==5.11.3 -pytz==2024.1 -PyYAML==6.0.2rc1 -rapidfuzz==3.9.3 -redis==5.0.4 -regex==2024.4.28 -requests==2.32.3 -requests-file==2.1.0 -requests-toolbelt==1.0.0 -rsa==4.9 -scramp==1.4.5 -setuptools==69.5.1 -simple-websocket==1.0.0 -six==1.16.0 -sniffio==1.3.1 -soupsieve==2.5 -speaklater==1.3 -SQLAlchemy==2.0.31 -tabulate==0.9.0 -tenacity==8.4.2 -tiktoken==0.7.0 -tldextract==5.1.2 -tokenizers==0.19.1 -tornado==6.4.1 -tqdm==4.66.4 -typing-inspect==0.9.0 -typing_extensions==4.12.2 -tzdata==2024.1 -unstructured==0.14.8 -unstructured-client==0.23.7 -urllib3==2.2.2 -uWSGI==2.0.26 -vine==5.1.0 -visitor==0.1.3 -wcwidth==0.2.13 -Werkzeug==3.0.3 -wrapt==1.16.0 -wsproto==1.2.0 -WTForms==3.1.2 -wtforms-html5==0.6.1 -yarl==1.9.4 -zope.event==5.0 -zope.interface==6.3 -zxcvbn==4.4.28 diff --git a/requirements_orig.txt b/requirements_orig.txt deleted file mode 100644 index e9b73d0..0000000 --- a/requirements_orig.txt +++ /dev/null @@ -1,19 +0,0 @@ -Flask~=3.0.3 -WTForms~=3.1.2 -SQLAlchemy~=2.0.29 -alembic~=1.13.1 -Werkzeug~=3.0.2 -pgvector~=0.2.5 -gevent~=24.2.1 -celery~=5.4.0 -kombu~=5.3.7 -langchain~=0.1.17 -requests~=2.31.0 -beautifulsoup4~=4.12.3 -google~=3.0.0 -redis~=5.0.4 -itsdangerous~=2.2.0 -pydantic~=2.7.1 -chardet~=5.2.0 -langcodes~=3.4.0 -pytz~=2024.1 \ No newline at end of file