diff --git a/common/utils/middleware.py b/common/utils/middleware.py index 8e1d16a..d1c3503 100644 --- a/common/utils/middleware.py +++ b/common/utils/middleware.py @@ -4,7 +4,7 @@ for handling tenant requests """ from flask_security import current_user -from flask import session +from flask import session, current_app from .database import Database @@ -17,12 +17,15 @@ def mw_before_request(): tenant_id = session['tenant']['id'] if not tenant_id: - return {"message": "You are not logged into any tenant"}, 403 + raise Exception('Cannot switch schema for tenant: no tenant defined in session') + + for role in current_user.roles: + current_app.logger.debug(f'In middleware: User {current_user.email} has role {role.name}') # user = User.query.get(current_user.id) - if current_user.has_roles(['Super User']) or current_user.tenant_id == tenant_id: + if current_user.has_role('Super User') or current_user.tenant_id == tenant_id: Database(tenant_id).switch_schema() else: - return {"message": "You are not a member of this tenant"}, 403 + raise Exception(f'Cannot switch schema for tenant {tenant_id}: user {current_user.email} does not have access') diff --git a/common/utils/model_utils.py b/common/utils/model_utils.py index e69de29..939e741 100644 --- a/common/utils/model_utils.py +++ b/common/utils/model_utils.py @@ -0,0 +1,79 @@ +from flask import current_app +from langchain.embeddings import OpenAIEmbeddings +from langchain.chat_models import ChatOpenAI +from langchain.prompts import ChatPromptTemplate + +from common.models.document import EmbeddingSmallOpenAI + + +def select_model_variables(tenant): + embedding_provider = tenant.embedding_model.rsplit('.', 1)[0] + embedding_model = tenant.embedding_model.rsplit('.', 1)[1] + + llm_provider = tenant.llm_model.rsplit('.', 1)[0] + llm_model = tenant.llm_model.rsplit('.', 1)[1] + + # Set model variables + model_variables = {} + if tenant.es_k: + model_variables['k'] = tenant.es_k + else: + model_variables['k'] = 5 + + if tenant.es_similarity_threshold: + model_variables['similarity_threshold'] = tenant.es_similarity_threshold + else: + model_variables['similarity_threshold'] = 0.7 + + if tenant.chat_RAG_temperature: + model_variables['RAG_temperature'] = tenant.chat_RAG_temperature + else: + model_variables['RAG_temperature'] = 0.3 + + if tenant.chat_no_RAG_temperature: + model_variables['no_RAG_temperature'] = tenant.chat_no_RAG_temperature + else: + model_variables['no_RAG_temperature'] = 0.5 + + # Set Embedding variables + match embedding_provider: + case 'openai': + match embedding_model: + case 'text-embedding-3-small': + api_key = current_app.config.get('OPENAI_API_KEY') + model_variables['embedding_model'] = OpenAIEmbeddings(api_key=api_key, + model='text-embedding-3-small') + model_variables['embedding_db_model'] = EmbeddingSmallOpenAI + model_variables['min_chunk_size'] = current_app.config.get('OAI_TE3S_MIN_CHUNK_SIZE') + model_variables['max_chunk_size'] = current_app.config.get('OAI_TE3S_MAX_CHUNK_SIZE') + case _: + raise Exception(f'Error setting model variables for tenant {tenant.id} ' + f'error: Invalid embedding model') + case _: + raise Exception(f'Error setting model variables for tenant {tenant.id} ' + f'error: Invalid embedding provider') + + # Set Chat model variables + match llm_provider: + case 'openai': + api_key = current_app.config.get('OPENAI_API_KEY') + model_variables['llm'] = ChatOpenAI(api_key=api_key, + model=llm_model, + temperature=model_variables['RAG_temperature']) + match llm_model: + case 'gpt-4-turbo' | 'gpt-4o': + summary_template = current_app.config.get('GPT4_SUMMARY_TEMPLATE') + rag_template = current_app.config.get('GPT4_RAG_TEMPLATE') + case 'gpt-3-5-turbo': + summary_template = current_app.config.get('GPT3_5_SUMMARY_TEMPLATE') + rag_template = current_app.config.get('GPT3_5_RAG_TEMPLATE') + case _: + raise Exception(f'Error setting model variables for tenant {tenant.id} ' + f'error: Invalid chat model') + model_variables['summary_prompt'] = ChatPromptTemplate.from_template(summary_template) + model_variables['rag_prompt'] = ChatPromptTemplate.from_template(rag_template) + case _: + raise Exception(f'Error setting model variables for tenant {tenant.id} ' + f'error: Invalid chat provider') + + return model_variables diff --git a/config/config.py b/config/config.py index aaa5486..6908a30 100644 --- a/config/config.py +++ b/config/config.py @@ -69,8 +69,8 @@ class Config(object): CELERY_ENABLE_UTC = True # Chunk Definition, Embedding dependent - O_TE3SMALL_MIN_CHUNK_SIZE = 2000 - O_TE3SMALL_MAX_CHUNK_SIZE = 3000 + OAI_TE3S_MIN_CHUNK_SIZE = 2000 + OAI_TE3S_MAX_CHUNK_SIZE = 3000 # LLM TEMPLATES GPT4_SUMMARY_TEMPLATE = """Write a concise summary of the text in the same language as the provided text. diff --git a/eveai_app/__init__.py b/eveai_app/__init__.py index b1c2768..b85bf39 100644 --- a/eveai_app/__init__.py +++ b/eveai_app/__init__.py @@ -70,7 +70,7 @@ def create_app(config_file=None): security_logger.setLevel(logging.DEBUG) sqlalchemy_logger = logging.getLogger('sqlalchemy.engine') sqlalchemy_logger.setLevel(logging.DEBUG) - log_request_middleware(app) # Add this when debugging nginx or another proxy + # log_request_middleware(app) # Add this when debugging nginx or another proxy # Some generic Error Handling Routines @app.errorhandler(Exception) diff --git a/eveai_app/templates/document/document_languages.html b/eveai_app/templates/document/document_languages.html new file mode 100644 index 0000000..5e7ef69 --- /dev/null +++ b/eveai_app/templates/document/document_languages.html @@ -0,0 +1,24 @@ +{% extends 'base.html' %} +{% from 'macros.html' import render_selectable_table, render_pagination %} + +{% block title %}Document Languages{% endblock %} + +{% block content_title %}Document Languages{% endblock %} +{% block content_description %}View Languages for {{ document }}{% endblock %} +{% block content_class %}
{% endblock %} + +{% block content %} +
+
+ {{ render_selectable_table(headers=["Document Language ID", "Language", "User Context", "System Context"], rows=rows, selectable=True, id="documentsTable") }} +
+ + +
+
+
+{% endblock %} + +{% block content_footer %} + {{ render_pagination(pagination, 'document_bp.documents') }} +{% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/document/document_versions.html b/eveai_app/templates/document/document_versions.html new file mode 100644 index 0000000..e02adc1 --- /dev/null +++ b/eveai_app/templates/document/document_versions.html @@ -0,0 +1,23 @@ +{% extends 'base.html' %} +{% from 'macros.html' import render_selectable_table, render_pagination %} + +{% block title %}Document Versions{% endblock %} + +{% block content_title %}Document Versions{% endblock %} +{% block content_description %}View Versions for {{ document }}{% endblock %} +{% block content_class %}
{% endblock %} + +{% block content %} +
+
+ {{ render_selectable_table(headers=["Document Version ID", "URL", "File Location", "File Name", "File Type", "Processing", "Processing Start", "Proceeing Finish"], rows=rows, selectable=True, id="versionsTable") }} +
+ +
+
+
+{% endblock %} + +{% block content_footer %} + {{ render_pagination(pagination, 'document_bp.documents') }} +{% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/document/documents.html b/eveai_app/templates/document/documents.html index 007fbd0..483e023 100644 --- a/eveai_app/templates/document/documents.html +++ b/eveai_app/templates/document/documents.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% from 'macros.html' import render_nested_table, render_pagination %} +{% from 'macros.html' import render_selectable_table, render_pagination %} {% block title %}Documents{% endblock %} @@ -9,8 +9,13 @@ {% block content %}
- - {{ render_nested_table(headers=["Name", "Created At", "Valid From", "Languages & Versions"], rows=rows) }} +
+ {{ render_selectable_table(headers=["Document ID", "Name", "Valid From", "Valid To"], rows=rows, selectable=True, id="documentsTable") }} +
+ + +
+
{% endblock %} diff --git a/eveai_app/templates/document/edit_document.html b/eveai_app/templates/document/edit_document.html new file mode 100644 index 0000000..7f8626a --- /dev/null +++ b/eveai_app/templates/document/edit_document.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} +{% from "macros.html" import render_field %} +{% block title %}Update Document{% endblock %} + +{% block content_title %}Update Document{% endblock %} +{% block content_description %}Update document details.{% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = [] %} + {% set exclude_fields = [] %} + {% for field in form %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + +
+{% endblock %} diff --git a/eveai_app/templates/document/edit_document_language.html b/eveai_app/templates/document/edit_document_language.html new file mode 100644 index 0000000..73d68e6 --- /dev/null +++ b/eveai_app/templates/document/edit_document_language.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} +{% from "macros.html" import render_field %} +{% block title %}Update Document Language{% endblock %} + +{% block content_title %}Update Document Language{% endblock %} +{% block content_description %}Update document language for {{ doc_details }}.{% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = ['language', 'system_context'] %} + {% set exclude_fields = [] %} + {% for field in form %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + +
+{% endblock %} diff --git a/eveai_app/templates/macros.html b/eveai_app/templates/macros.html index ff39cf0..4a26eb7 100644 --- a/eveai_app/templates/macros.html +++ b/eveai_app/templates/macros.html @@ -217,57 +217,6 @@
{% endmacro %} -{% macro render_seamless_table(headers, rows) %} -{% macro render_integrated_table(headers, data) %} -
- - - - {% for header in headers %} - - {% endfor %} - - - - {% for entry in data %} - {% if entry.is_group and entry.sub_rows %} - {% for sub_row in entry.sub_rows %} - - {% for cell in sub_row %} - {% if cell %} - - {% else %} - - {% endif %} - {% endfor %} - - {% endfor %} - {% else %} - - {% for cell in entry %} - - {% endfor %} - - {% endif %} - {% endfor %} - -
{{ header }}
- {% if cell.type == 'text' %} -

{{ cell.value }}

- {% else %} - {{ cell.value }} - {% endif %} -
- {% if cell.type == 'text' %} -

{{ cell.value }}

- {% else %} - {{ cell.value }} - {% endif %} -
-
-{% endmacro %} -{% endmacro %} - {% macro render_pagination(pagination, endpoint) %}