From 659588deab848ab50979d3727552f19c1afeb758 Mon Sep 17 00:00:00 2001 From: Josako Date: Thu, 2 May 2024 00:12:27 +0200 Subject: [PATCH] Changes Documents - llm and languagefields on tenant, processing on documents first version of Adding Documents (excl. embeddings) --- config.py | 9 ++ eveai_app/__init__.py | 2 + eveai_app/models/document.py | 8 +- eveai_app/models/user.py | 19 ++++- .../templates/basic/session_defaults.html | 21 +++++ .../templates/document/add_document.html | 24 ++++++ eveai_app/templates/navbar.html | 85 ++++++++++++++----- eveai_app/utils/app_hooks.py | 12 +++ eveai_app/utils/middleware.py | 5 +- eveai_app/utils/security.py | 4 + eveai_app/views/__init__.py | 4 +- eveai_app/views/basic_forms.py | 30 +++++++ eveai_app/views/basic_views.py | 21 +++-- eveai_app/views/document_forms.py | 4 +- eveai_app/views/document_views.py | 65 ++++++++++++-- eveai_app/views/user_forms.py | 29 ++++++- eveai_app/views/user_views.py | 40 ++++++++- 17 files changed, 331 insertions(+), 51 deletions(-) create mode 100644 eveai_app/templates/basic/session_defaults.html create mode 100644 eveai_app/templates/document/add_document.html create mode 100644 eveai_app/utils/app_hooks.py create mode 100644 eveai_app/views/basic_forms.py diff --git a/config.py b/config.py index 94e4b69..3cc79d4 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,5 @@ from os import environ, path +from datetime import timedelta basedir = path.abspath(path.dirname(__file__)) @@ -20,6 +21,8 @@ class Config(object): SECURITY_POST_LOGIN_VIEW = '/user/tenant' SECURITY_RECOVERABLE = True SECURITY_EMAIL_SENDER = "eveai_super@flow-it.net" + PERMANENT_SESSION_LIFETIME = timedelta(minutes=60) + # flask-mailman settings MAIL_SERVER = 'mail.flow-it.net' @@ -32,6 +35,12 @@ class Config(object): MAX_CONTENT_LENGTH = 16 * 1024 * 1024 UPLOAD_EXTENSIONS = ['.txt', '.pdf', '.png', '.jpg', '.jpeg', '.gif'] + # supported languages + SUPPORTED_LANGUAGES = ['en', 'fr', 'nl', 'de', 'es'] + + # supported LLMs + SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'mistral.mistral-embed'] + SUPPORTED_LLMS = ['openai.gpt-4-turbo', 'openai.gpt-3.5-turbo', 'mistral.mistral-large-2402'] class DevConfig(Config): DEVELOPMENT = True diff --git a/eveai_app/__init__.py b/eveai_app/__init__.py index 7f6ffff..ba47248 100644 --- a/eveai_app/__init__.py +++ b/eveai_app/__init__.py @@ -65,6 +65,8 @@ def register_blueprints(app): app.register_blueprint(user_bp) from .views.basic_views import basic_bp app.register_blueprint(basic_bp) + from .views.document_views import document_bp + app.register_blueprint(document_bp) def register_api(app): diff --git a/eveai_app/models/document.py b/eveai_app/models/document.py index 557a5d6..eb06a0b 100644 --- a/eveai_app/models/document.py +++ b/eveai_app/models/document.py @@ -55,6 +55,12 @@ class DocumentVersion(db.Model): updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now()) updated_by = db.Column(db.Integer, db.ForeignKey(User.id)) + # Processing Information + processing = db.Column(db.Boolean, nullable=False, default=False) + processing_started_at = db.Column(db.DateTime, nullable=True) + processing_finished_at = db.Column(db.DateTime, nullable=True) + processing_error = db.Column(db.String(255), nullable=True) + # Relations embeddings = db.relationship('EmbeddingMistral', backref='document_version', lazy=True) @@ -62,7 +68,7 @@ class DocumentVersion(db.Model): return f".{self.id}>" def calc_file_location(self): - return f"{self.document_language.document.tenant.id}/{self.document_language.document.id}/{self.document_language.language}" + return f"{self.document_language.document.tenant_id}/{self.document_language.document.id}/{self.document_language.language}" def calc_file_name(self): return f"{self.id}.{self.file_type}" diff --git a/eveai_app/models/user.py b/eveai_app/models/user.py index 7aa3c78..62e57af 100644 --- a/eveai_app/models/user.py +++ b/eveai_app/models/user.py @@ -1,5 +1,7 @@ from ..extensions import db from flask_security import UserMixin, RoleMixin +from sqlalchemy.dialects.postgresql import ARRAY +import sqlalchemy as sa class Tenant(db.Model): @@ -19,10 +21,13 @@ class Tenant(db.Model): # language information default_language = db.Column(db.String(2), nullable=True) + allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True) - # currency information - default_currency = db.Column(db.String(80), nullable=True) - + # LLM specific choices + default_embedding_model = db.Column(db.String(50), nullable=True) + allowed_embedding_models = db.Column(ARRAY(sa.String(50)), nullable=True) + default_llm_model = db.Column(db.String(50), nullable=True) + allowed_llm_models = db.Column(ARRAY(sa.String(50)), nullable=True) # Licensing Information license_start_date = db.Column(db.Date, nullable=True) @@ -33,13 +38,19 @@ class Tenant(db.Model): users = db.relationship('User', backref='tenant') def __repr__(self): - return '' % self.name + return f"" def to_dict(self): return { 'id': self.id, 'name': self.name, 'website': self.website, + 'default_language': self.default_language, + 'allowed_languages': self.allowed_languages, + 'default_embedding_model': self.default_embedding_model, + 'allowed_embedding_models': self.allowed_embedding_models, + 'default_llm_model': self.default_llm_model, + 'allowed_llm_models': self.allowed_llm_models, 'license_start_date': self.license_start_date, 'license_end_date': self.license_end_date, 'allowed_monthly_interactions': self.allowed_monthly_interactions diff --git a/eveai_app/templates/basic/session_defaults.html b/eveai_app/templates/basic/session_defaults.html new file mode 100644 index 0000000..5e94509 --- /dev/null +++ b/eveai_app/templates/basic/session_defaults.html @@ -0,0 +1,21 @@ +{% extends 'base.html' %} +{% from "macros.html" import render_field %} + +{% block title %}Session Defaults{% endblock %} + +{% block content_title %}Session Defaults{% endblock %} +{% block content_description %}Specifiy the defaults to be used in your session.{% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = ['user_name', 'user_email', 'tenant_name'] %} + {% set exclude_fields = [] %} + {% for field in form %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + +
+{% endblock %} + +{% block content_footer %} {% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/document/add_document.html b/eveai_app/templates/document/add_document.html new file mode 100644 index 0000000..dcaeb1c --- /dev/null +++ b/eveai_app/templates/document/add_document.html @@ -0,0 +1,24 @@ +{% extends 'base.html' %} +{% from "macros.html" import render_field %} + +{% block title %}Add Document{% endblock %} + +{% block content_title %}Add Document{% endblock %} +{% block content_description %}Add a document to EveAI{% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = [] %} + {% set exclude_fields = [] %} + {% for field in form %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + +
+{% endblock %} + + +{% block content_footer %} + +{% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/navbar.html b/eveai_app/templates/navbar.html index cdcec0b..84eb219 100644 --- a/eveai_app/templates/navbar.html +++ b/eveai_app/templates/navbar.html @@ -42,26 +42,69 @@ {% endmacro %} -
-