pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changes Documents - llm and languagefields on tenant, processing on documents

Browse Source 
first version of Adding Documents (excl. embeddings)
This commit is contained in:
Josako
2024-05-02 00:12:27 +02:00
parent 8e4e4d8586
commit 659588deab
17 changed files with 331 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
config.py
View File

@@ -1,4 +1,5 @@
from os import environ, path
from datetime import timedelta
basedir = path.abspath(path.dirname(__file__))
@@ -20,6 +21,8 @@ class Config(object):
SECURITY_POST_LOGIN_VIEW = '/user/tenant'
SECURITY_RECOVERABLE = True
SECURITY_EMAIL_SENDER = "eveai_super@flow-it.net"
PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
# flask-mailman settings
MAIL_SERVER = 'mail.flow-it.net'
@@ -32,6 +35,12 @@ class Config(object):
MAX_CONTENT_LENGTH = 16 * 1024 * 1024
UPLOAD_EXTENSIONS = ['.txt', '.pdf', '.png', '.jpg', '.jpeg', '.gif']
# supported languages
SUPPORTED_LANGUAGES = ['en', 'fr', 'nl', 'de', 'es']
# supported LLMs
SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'mistral.mistral-embed']
SUPPORTED_LLMS = ['openai.gpt-4-turbo', 'openai.gpt-3.5-turbo', 'mistral.mistral-large-2402']
class DevConfig(Config):
DEVELOPMENT = True

2
eveai_app/__init__.py
View File

@@ -65,6 +65,8 @@ def register_blueprints(app):
app.register_blueprint(user_bp)
from .views.basic_views import basic_bp
app.register_blueprint(basic_bp)
from .views.document_views import document_bp
app.register_blueprint(document_bp)
def register_api(app):

8
eveai_app/models/document.py
View File

@@ -55,6 +55,12 @@ class DocumentVersion(db.Model):
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now())
updated_by = db.Column(db.Integer, db.ForeignKey(User.id))
# Processing Information
processing = db.Column(db.Boolean, nullable=False, default=False)
processing_started_at = db.Column(db.DateTime, nullable=True)
processing_finished_at = db.Column(db.DateTime, nullable=True)
processing_error = db.Column(db.String(255), nullable=True)
# Relations
embeddings = db.relationship('EmbeddingMistral', backref='document_version', lazy=True)
@@ -62,7 +68,7 @@ class DocumentVersion(db.Model):
return f"<DocumentVersion {self.document_language.document_id}.{self.document_language.language}>.{self.id}>"
def calc_file_location(self):
return f"{self.document_language.document.tenant.id}/{self.document_language.document.id}/{self.document_language.language}"
return f"{self.document_language.document.tenant_id}/{self.document_language.document.id}/{self.document_language.language}"
def calc_file_name(self):
return f"{self.id}.{self.file_type}"

19
eveai_app/models/user.py
View File

@@ -1,5 +1,7 @@
from ..extensions import db
from flask_security import UserMixin, RoleMixin
from sqlalchemy.dialects.postgresql import ARRAY
import sqlalchemy as sa
class Tenant(db.Model):
@@ -19,10 +21,13 @@ class Tenant(db.Model):
# language information
default_language = db.Column(db.String(2), nullable=True)
allowed_languages = db.Column(ARRAY(sa.String(2)), nullable=True)
# currency information
default_currency = db.Column(db.String(80), nullable=True)
# LLM specific choices
default_embedding_model = db.Column(db.String(50), nullable=True)
allowed_embedding_models = db.Column(ARRAY(sa.String(50)), nullable=True)
default_llm_model = db.Column(db.String(50), nullable=True)
allowed_llm_models = db.Column(ARRAY(sa.String(50)), nullable=True)
# Licensing Information
license_start_date = db.Column(db.Date, nullable=True)
@@ -33,13 +38,19 @@ class Tenant(db.Model):
users = db.relationship('User', backref='tenant')
def __repr__(self):
return '<Tenant %r>' % self.name
return f"<Tenant {self.id}: {self.name}>"
def to_dict(self):
return {
'id': self.id,
'name': self.name,
'website': self.website,
'default_language': self.default_language,
'allowed_languages': self.allowed_languages,
'default_embedding_model': self.default_embedding_model,
'allowed_embedding_models': self.allowed_embedding_models,
'default_llm_model': self.default_llm_model,
'allowed_llm_models': self.allowed_llm_models,
'license_start_date': self.license_start_date,
'license_end_date': self.license_end_date,
'allowed_monthly_interactions': self.allowed_monthly_interactions

21
eveai_app/templates/basic/session_defaults.html Normal file
View File

@@ -0,0 +1,21 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% block title %}Session Defaults{% endblock %}
{% block content_title %}Session Defaults{% endblock %}
{% block content_description %}Specifiy the defaults to be used in your session.{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = ['user_name', 'user_email', 'tenant_name'] %}
{% set exclude_fields = [] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Set Session Defaults</button>
</form>
{% endblock %}
{% block content_footer %} {% endblock %}

24
eveai_app/templates/document/add_document.html Normal file
View File

@@ -0,0 +1,24 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% block title %}Add Document{% endblock %}
{% block content_title %}Add Document{% endblock %}
{% block content_description %}Add a document to EveAI{% endblock %}
{% block content %}
<form method="post" enctype="multipart/form-data">
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Add Document</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

85
eveai_app/templates/navbar.html
View File

@@ -42,26 +42,69 @@
</li>
{% endmacro %}
<div class="navbar navbar-expand-lg navbar-light bg-white z-index-3 py-3">
<div class="collapse navbar-collapse w-100 pt-3 pb-2 py-lg-0" id="navigation">
<ul class="navbar-nav navbar-nav-hover mx-auto">
{% if current_user.is_authenticated %}
{{ dropdown('User Mgmt', 'contacts', [
{'name': 'Select Tenant', 'url': '/user/select_tenant', 'roles': ['Super User']},
{'name': 'Tenant Registration', 'url': '/user/tenant', 'roles': ['Super User']},
{'name': 'User Registration', 'url': '/user/user', 'roles': ['admin', 'manager']},
{'name': 'User List', 'url': '/user/view_users/' + session['tenant']['id']|string, 'roles': ['Super User', 'Tenant Admin']}
]) }}
{% endif %}
{{ dropdown('Account', 'contacts', [
{'name': 'Login', 'url': '/login'},
{'name': 'Logout', 'url': '/logout'}
]) }}
{% if current_user.is_authenticated %}
{% if 'tenant' in session %}
<li>TENANT ID: {{ session['tenant'].get('id', 'None') }}, TENANT NAME: {{ session['tenant'].get('name', 'None') }}</li>
{% endif %}
{% endif %}
</ul>
<div class="container position-sticky z-index-sticky top-0">
<div class="row">
<div class="col-12">
<nav class="navbar navbar-expand-lg navbar-light bg-white z-index-3 py-3">
<div class="container-fluid px-0">
<a class="navbar-brand font-weight-bolder ms-sm-3 d-none d-md-block" href=" https://www.flow-it.net " rel="tooltip" title="Realised by Josako & Kobe" data-placement="bottom" target="_blank">
EveAI
</a>
<a class="navbar-brand font-weight-bolder ms-sm-3 d-block d-md-none" href=" https://www.flow-it.net " rel="tooltip" title="Realised by Josako & Kobe" data-placement="bottom" target="_blank">
EveAI
</a>
<a href="/session_defaults" class="btn btn-sm bg-gradient-primary mb-0 ms-auto d-lg-none d-block">
{% if 'tenant' in session %}
TENANT: {{ session['tenant'].get('name', 'None') }}
{% endif %}
</a>
<button class="navbar-toggler shadow-none ms-md-2" type="button" data-bs-toggle="collapse" data-bs-target="#navigation" aria-controls="navigation" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon mt-2">
<span class="navbar-toggler-bar bar1"></span>
<span class="navbar-toggler-bar bar2"></span>
<span class="navbar-toggler-bar bar3"></span>
</span>
</button>
<div class="collapse navbar-collapse w-100 pt-3 pb-2 py-lg-0" id="navigation">
<ul class="navbar-nav navbar-nav-hover mx-auto">
{% if current_user.is_authenticated %}
{{ dropdown('User Mgmt', 'contacts', [
{'name': 'Select Tenant', 'url': '/user/select_tenant', 'roles': ['Super User']},
{'name': 'Tenant Registration', 'url': '/user/tenant', 'roles': ['Super User']},
{'name': 'User Registration', 'url': '/user/user', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'User List', 'url': '/user/view_users/' + session['tenant']['id']|string, 'roles': ['Super User', 'Tenant Admin']}
]) }}
{% endif %}
{% if current_user.is_authenticated %}
{{ dropdown('Document Mgmt', 'contacts', [
{'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Tenant Admin']},
]) }}
{% endif %}
{% if current_user.is_authenticated %}
{{ dropdown(current_user.user_name, 'contacts', [
{'name': 'Session Defaults', 'url': '/session_defaults', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Logout', 'url': '/logout'}
]) }}
{% else %}
{{ dropdown('Account', 'contacts', [
{'name': 'Login', 'url': '/login'}
]) }}
{% endif %}
</ul>
{% if current_user.is_authenticated %}
<ul class="navbar-nav d-lg-block d-none">
<li class="nav-item">
<a href="/session_defaults" class="btn btn-sm bg-gradient-primary mb-0">
{% if 'tenant' in session %}
TENANT: {{ session['tenant'].get('name', 'None') }}
{% endif %}
</a>
</li>
</ul>
{% endif %}
</div>
</div>
</nav>
</div>
</div>
</div>

12
eveai_app/utils/app_hooks.py Normal file
View File

@@ -0,0 +1,12 @@
from flask import session
def make_session_permanent():
session.permanent = True # Refresh the session timeout on every request
# if 'user_id' in session and session.get('was_authenticated'):
# if session.modified: # Check if the session was modified
# session['was_authenticated'] = True
# else:
# session.pop('user_id', None) # Clear session
# session.pop('was_authenticated', None)
# return redirect(url_for('login')) # Redirect to login page if session expired

5
eveai_app/utils/middleware.py
View File

@@ -8,7 +8,6 @@ from flask import session
from ..models.user import User, Tenant
from .database import Database
from ..views.document_views import document_bp
def mw_before_request():
@@ -21,8 +20,8 @@ def mw_before_request():
if not tenant_id:
return {"message": "You are not logged into any tenant"}, 403
user = User.get_by_id(current_user.id)
if user.has_roles(['Super User']) or user.tenant_id == tenant_id:
# user = User.query.get(current_user.id)
if current_user.has_roles(['Super User']) or current_user.tenant_id == tenant_id:
Database(tenant_id).switch_schema()
else:
return {"message": "You are not a member of this tenant"}, 403

4
eveai_app/utils/security.py
View File

@@ -6,3 +6,7 @@ from ..models.user import User, Tenant
def set_tenant_session_data(sender, user, **kwargs):
tenant = Tenant.query.filter_by(id=user.tenant_id).first()
session['tenant'] = tenant.to_dict()
session['default_language'] = tenant.default_language
session['default_embedding_model'] = tenant.default_embedding_model
session['default_llm_model'] = tenant.default_llm_model

4
eveai_app/views/__init__.py
View File

@@ -1,5 +1,5 @@
import eveai_app.views.user_views
# import eveai_app.views.user_views
# import eveai_app.views.document_views
# document_bp = Blueprint('document_bp', __name__, url_prefix='document')
# interaction_bp = Blueprint('interaction_bp', __name__, url_prefix='interaction')

30
eveai_app/views/basic_forms.py Normal file
View File

@@ -0,0 +1,30 @@
from flask import session
from flask_security import current_user
from flask_wtf import FlaskForm
from wtforms import StringField, SelectField
from wtforms.validators import DataRequired
class SessionDefaultsForm(FlaskForm):
user_name = StringField('User Name', validators=[DataRequired()])
user_email = StringField('User', validators=[DataRequired()])
# Tenant Defaults
tenant_name = StringField('Tenant Name', validators=[DataRequired()])
default_language = SelectField(u'Default Language', choices=[], validators=[DataRequired()])
default_embedding_model = SelectField(u'Default Embedding Model', choices=[], validators=[DataRequired()])
default_llm_model = SelectField(u'Default LLM Model', choices=[], validators=[DataRequired()])
def __init__(self):
super(SessionDefaultsForm, self).__init__()
self.user_name.data = current_user.user_name
self.user_email.data = current_user.email
self.tenant_name.data = session.get('tenant').get('name')
self.default_language.choices = [(lang, lang.lower()) for lang in
session.get('tenant').get('allowed_languages')]
self.default_language.data = session.get('default_language')
self.default_embedding_model.choices = [(model, model) for model in
session.get('tenant').get('allowed_embedding_models')]
self.default_embedding_model.data = session.get('default_embedding_model')
self.default_llm_model.choices = [(model, model) for model in session.get('tenant').get('allowed_llm_models')]
self.default_llm_model.data = session.get('default_llm_model')

21
eveai_app/views/basic_views.py
View File

@@ -1,10 +1,7 @@
from flask import request, redirect, url_for, flash, render_template, Blueprint, session
from flask_security import hash_password, roles_required, roles_accepted
from flask_security import roles_required, roles_accepted
from ..models.user import User, Tenant, Role
from ..extensions import db
from .user_forms import TenantForm, CreateUserForm, EditUserForm
from ..utils.database import Database
from .basic_forms import SessionDefaultsForm
basic_bp = Blueprint('basic_bp', __name__)
@@ -12,3 +9,17 @@ basic_bp = Blueprint('basic_bp', __name__)
@basic_bp.route('/', methods=['GET', ])
def index():
return render_template('index.html')
@basic_bp.route('/session_defaults', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def session_defaults():
form = SessionDefaultsForm()
if form.validate_on_submit():
session['default_language'] = form.default_language.data
session['default_embedding_model'] = form.default_embedding_model.data
session['default_llm_model'] = form.default_llm_model.data
return render_template('basic/session_defaults.html', form=form)

4
eveai_app/views/document_forms.py
View File

@@ -1,7 +1,7 @@
from flask_wtf import FlaskForm
from wtforms import (StringField, BooleanField, SubmitField, DateField,
SelectMultipleField, FieldList, FormField)
from wtforms.validators import DataRequired, Length
from wtforms.validators import DataRequired, Length, Optional
from flask_wtf.file import FileField, FileAllowed, FileRequired
@@ -10,6 +10,6 @@ class AddDocumentForm(FlaskForm):
FileRequired()])
name = StringField('Name', validators=[Length(max=100)])
language = StringField('Language', validators=[Length(max=2)])
valid_from = DateField('Valid from', id='form-control datepicker')
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
submit = SubmitField('Submit')

65
eveai_app/views/document_views.py
View File

@@ -22,6 +22,9 @@ def before_request():
@roles_accepted('Super User', 'Tenant Admin')
def add_document():
form = AddDocumentForm()
error = None
# If the form is submitted
if request.method == 'POST' and form.validate_on_submit():
file = form.file.data
filename = secure_filename(file.filename)
@@ -39,17 +42,69 @@ def add_document():
new_doc.valid_from = form.valid_from.data
else:
new_doc.valid_from = timestamp
new_doc.tenant_id = session['tenant']['id']
new_doc.created_at = timestamp
new_doc.updated_at = timestamp
new_doc.created_by = current_user.id
new_doc.updated_by = current_user.id
db.session.add(new_doc)
# TODO: Continue from here on and complete add_document
# Create the DocumentLanguage
new_doc_lang = DocumentLanguage()
language = form.language.data
if form.language.data == '':
new_doc_lang.language = session['default_language']
else:
new_doc_lang.language = form.language.data
new_doc_lang.document = new_doc
file.save(os.path.join(current_app.config['UPLOAD_FOLDER'], filename))
return redirect(url_for('document_bp.add_document'))
new_doc_lang.created_at = timestamp
new_doc_lang.updated_at = timestamp
new_doc_lang.created_by = current_user.id
new_doc_lang.updated_by = current_user.id
# Create the DocumentVersion
new_doc_vers = DocumentVersion()
new_doc_vers.document_language = new_doc_lang
new_doc_vers.created_at = timestamp
new_doc_vers.updated_at = timestamp
new_doc_vers.created_by = current_user.id
new_doc_vers.updated_by = current_user.id
try:
db.session.add(new_doc)
db.session.add(new_doc_lang)
db.session.add(new_doc_vers)
db.session.commit()
except Exception as e:
db.session.rollback()
error = e.args
# Save the file and process the document
if error is None:
flash('Document added successfully.')
new_doc_vers.file_type = extension
new_doc_vers.file_name = new_doc_vers.calc_file_name()
new_doc_vers.file_location = new_doc_vers.calc_file_location()
upload_path = os.path.join(current_app.config['UPLOAD_FOLDER'], new_doc_vers.file_location)
if not os.path.exists(upload_path):
os.makedirs(upload_path, exist_ok=True)
file.save(os.path.join(upload_path, new_doc_vers.file_name))
try:
db.session.commit()
except Exception as e:
db.session.rollback()
error = e.args
if error is None:
flash('Document saved successfully.')
# TODO: processing of document to embeddings (async)
flash('Document processing started.')
else:
flash('Error saving document.')
else:
flash('Error adding document.')
return render_template('document/add_document.html', form=form)

29
eveai_app/views/user_forms.py
View File

@@ -1,18 +1,39 @@
from flask import current_app
from flask_wtf import FlaskForm
from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
SelectMultipleField, FieldList, FormField)
from wtforms.validators import DataRequired, Length, Email, NumberRange
SelectField, SelectMultipleField, FieldList, FormField)
from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional
from ..models.user import User, Role
class TenantForm(FlaskForm):
name = StringField('Name', validators=[DataRequired(), Length(max=80)])
website = StringField('Website', validators=[DataRequired(), Length(max=255)])
license_start_date = DateField('License Start Date', id='form-control datepicker')
license_end_date = DateField('License End Date', id='datepicker')
# language fields
default_language = SelectField('Default Language', choices=[], validators=[DataRequired()])
allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[DataRequired()])
# LLM fields
default_embedding_model = SelectField('Default Embedding Model', choices=[], validators=[DataRequired()])
allowed_embedding_models = SelectMultipleField('Allowed Embedding Models', choices=[], validators=[DataRequired()])
default_llm_model = SelectField('Default Large Language Model', choices=[], validators=[DataRequired()])
allowed_llm_models = SelectMultipleField('Allowed Large Language Models', choices=[], validators=[DataRequired()])
# license fields
license_start_date = DateField('License Start Date', id='form-control datepicker', validators=[Optional()])
license_end_date = DateField('License End Date', id='form-control datepicker', validators=[Optional()])
allowed_monthly_interactions = IntegerField('Allowed Monthly Interactions', validators=[NumberRange(min=0)])
submit = SubmitField('Submit')
def __init__(self, *args, **kwargs):
super(TenantForm, self).__init__(*args, **kwargs)
# initialise language fields
self.default_language.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
self.allowed_languages.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
# initialise LLM fields
self.default_embedding_model.choices = [(model, model) for model in current_app.config['SUPPORTED_EMBEDDINGS']]
self.allowed_embedding_models.choices = [(model, model) for model in current_app.config['SUPPORTED_EMBEDDINGS']]
self.default_llm_model.choices = [(model, model) for model in current_app.config['SUPPORTED_LLMS']]
self.allowed_llm_models.choices = [(model, model) for model in current_app.config['SUPPORTED_LLMS']]
class BaseUserForm(FlaskForm):
user_name = StringField('Name', validators=[DataRequired(), Length(max=80)])

40
eveai_app/views/user_views.py
View File

@@ -35,6 +35,33 @@ def tenant():
lic_end = request.form.get('license_end_date')
monthly = request.form.get('allowed_monthly_interactions')
# language fields
default_language = request.form.get('default_language')
allowed_languages = request.form.getlist('allowed_languages')
if default_language != '':
new_tenant.default_language = default_language
if allowed_languages != '':
new_tenant.allowed_languages = allowed_languages
# LLM fields
default_embedding = request.form.get('default_embedding')
allowed_embeddings = request.form.getlist('allowed_embeddings')
if default_embedding != '':
new_tenant.default_embedding = default_embedding
if allowed_embeddings != '':
new_tenant.allowed_embeddings = allowed_embeddings
default_llm_model = request.form.get('default_llm_model')
allowed_llm_models = request.form.getlist('allowed_llm_models')
if default_llm_model != '':
new_tenant.default_llm_model = default_llm_model
if allowed_llm_models != '':
new_tenant.allowed_llm_models = allowed_llm_models
# license data
if lic_start != '':
new_tenant.license_start_date = dt.strptime(lic_start, '%Y-%m-%d')
if lic_end != '':
@@ -74,8 +101,12 @@ def edit_tenant(tenant_id):
# Populate the tenant with form data
form.populate_obj(tenant)
db.session.commit()
flash('User updated successfully.', 'success')
return redirect(url_for(f"user/tenant/tenant_id"))
print(session)
flash('Tenant updated successfully.', 'success')
if session.get('tenant'):
if session['tenant'].get('id') == tenant_id:
session['tenant'] = tenant.to_dict()
# return redirect(url_for(f"user/tenant/tenant_id"))
return render_template('user/edit_tenant.html', form=form, tenant_id=tenant_id)
@@ -180,8 +211,9 @@ def handle_tenant_selection():
tenant_id = request.form['tenant_id']
the_tenant = Tenant.query.get(tenant_id)
session['tenant'] = the_tenant.to_dict()
print("SESSION TENANT")
print(session['tenant'])
session['default_language'] = the_tenant.default_language
session['default_embedding_model'] = the_tenant.default_embedding_model
session['default_llm_model'] = the_tenant.default_llm_model
action = request.form['action']
if action == 'view_users':