Changes Documents - llm and languagefields on tenant, processing on documents
first version of Adding Documents (excl. embeddings)
This commit is contained in:
Josako
@@ -1,5 +1,5 @@
|
||||
import eveai_app.views.user_views
|
||||
|
||||
# import eveai_app.views.user_views
|
||||
# import eveai_app.views.document_views
|
||||
|
||||
# document_bp = Blueprint('document_bp', __name__, url_prefix='document')
|
||||
# interaction_bp = Blueprint('interaction_bp', __name__, url_prefix='interaction')
|
||||
|
||||
30
eveai_app/views/basic_forms.py
Normal file
30
eveai_app/views/basic_forms.py
Normal file
@@ -0,0 +1,30 @@
|
||||
from flask import session
|
||||
from flask_security import current_user
|
||||
from flask_wtf import FlaskForm
|
||||
from wtforms import StringField, SelectField
|
||||
from wtforms.validators import DataRequired
|
||||
|
||||
|
||||
class SessionDefaultsForm(FlaskForm):
|
||||
user_name = StringField('User Name', validators=[DataRequired()])
|
||||
user_email = StringField('User', validators=[DataRequired()])
|
||||
|
||||
# Tenant Defaults
|
||||
tenant_name = StringField('Tenant Name', validators=[DataRequired()])
|
||||
default_language = SelectField(u'Default Language', choices=[], validators=[DataRequired()])
|
||||
default_embedding_model = SelectField(u'Default Embedding Model', choices=[], validators=[DataRequired()])
|
||||
default_llm_model = SelectField(u'Default LLM Model', choices=[], validators=[DataRequired()])
|
||||
|
||||
def __init__(self):
|
||||
super(SessionDefaultsForm, self).__init__()
|
||||
self.user_name.data = current_user.user_name
|
||||
self.user_email.data = current_user.email
|
||||
self.tenant_name.data = session.get('tenant').get('name')
|
||||
self.default_language.choices = [(lang, lang.lower()) for lang in
|
||||
session.get('tenant').get('allowed_languages')]
|
||||
self.default_language.data = session.get('default_language')
|
||||
self.default_embedding_model.choices = [(model, model) for model in
|
||||
session.get('tenant').get('allowed_embedding_models')]
|
||||
self.default_embedding_model.data = session.get('default_embedding_model')
|
||||
self.default_llm_model.choices = [(model, model) for model in session.get('tenant').get('allowed_llm_models')]
|
||||
self.default_llm_model.data = session.get('default_llm_model')
|
||||
@@ -1,10 +1,7 @@
|
||||
from flask import request, redirect, url_for, flash, render_template, Blueprint, session
|
||||
from flask_security import hash_password, roles_required, roles_accepted
|
||||
from flask_security import roles_required, roles_accepted
|
||||
|
||||
from ..models.user import User, Tenant, Role
|
||||
from ..extensions import db
|
||||
from .user_forms import TenantForm, CreateUserForm, EditUserForm
|
||||
from ..utils.database import Database
|
||||
from .basic_forms import SessionDefaultsForm
|
||||
|
||||
basic_bp = Blueprint('basic_bp', __name__)
|
||||
|
||||
@@ -12,3 +9,17 @@ basic_bp = Blueprint('basic_bp', __name__)
|
||||
@basic_bp.route('/', methods=['GET', ])
|
||||
def index():
|
||||
return render_template('index.html')
|
||||
|
||||
|
||||
@basic_bp.route('/session_defaults', methods=['GET', 'POST'])
|
||||
@roles_accepted('Super User', 'Tenant Admin')
|
||||
def session_defaults():
|
||||
form = SessionDefaultsForm()
|
||||
|
||||
if form.validate_on_submit():
|
||||
session['default_language'] = form.default_language.data
|
||||
session['default_embedding_model'] = form.default_embedding_model.data
|
||||
session['default_llm_model'] = form.default_llm_model.data
|
||||
|
||||
return render_template('basic/session_defaults.html', form=form)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from flask_wtf import FlaskForm
|
||||
from wtforms import (StringField, BooleanField, SubmitField, DateField,
|
||||
SelectMultipleField, FieldList, FormField)
|
||||
from wtforms.validators import DataRequired, Length
|
||||
from wtforms.validators import DataRequired, Length, Optional
|
||||
from flask_wtf.file import FileField, FileAllowed, FileRequired
|
||||
|
||||
|
||||
@@ -10,6 +10,6 @@ class AddDocumentForm(FlaskForm):
|
||||
FileRequired()])
|
||||
name = StringField('Name', validators=[Length(max=100)])
|
||||
language = StringField('Language', validators=[Length(max=2)])
|
||||
valid_from = DateField('Valid from', id='form-control datepicker')
|
||||
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
|
||||
|
||||
submit = SubmitField('Submit')
|
||||
|
||||
@@ -22,6 +22,9 @@ def before_request():
|
||||
@roles_accepted('Super User', 'Tenant Admin')
|
||||
def add_document():
|
||||
form = AddDocumentForm()
|
||||
error = None
|
||||
|
||||
# If the form is submitted
|
||||
if request.method == 'POST' and form.validate_on_submit():
|
||||
file = form.file.data
|
||||
filename = secure_filename(file.filename)
|
||||
@@ -39,17 +42,69 @@ def add_document():
|
||||
new_doc.valid_from = form.valid_from.data
|
||||
else:
|
||||
new_doc.valid_from = timestamp
|
||||
|
||||
new_doc.tenant_id = session['tenant']['id']
|
||||
|
||||
new_doc.created_at = timestamp
|
||||
new_doc.updated_at = timestamp
|
||||
new_doc.created_by = current_user.id
|
||||
new_doc.updated_by = current_user.id
|
||||
db.session.add(new_doc)
|
||||
|
||||
# TODO: Continue from here on and complete add_document
|
||||
# Create the DocumentLanguage
|
||||
new_doc_lang = DocumentLanguage()
|
||||
language = form.language.data
|
||||
if form.language.data == '':
|
||||
new_doc_lang.language = session['default_language']
|
||||
else:
|
||||
new_doc_lang.language = form.language.data
|
||||
|
||||
new_doc_lang.document = new_doc
|
||||
|
||||
file.save(os.path.join(current_app.config['UPLOAD_FOLDER'], filename))
|
||||
return redirect(url_for('document_bp.add_document'))
|
||||
new_doc_lang.created_at = timestamp
|
||||
new_doc_lang.updated_at = timestamp
|
||||
new_doc_lang.created_by = current_user.id
|
||||
new_doc_lang.updated_by = current_user.id
|
||||
|
||||
# Create the DocumentVersion
|
||||
new_doc_vers = DocumentVersion()
|
||||
new_doc_vers.document_language = new_doc_lang
|
||||
|
||||
new_doc_vers.created_at = timestamp
|
||||
new_doc_vers.updated_at = timestamp
|
||||
new_doc_vers.created_by = current_user.id
|
||||
new_doc_vers.updated_by = current_user.id
|
||||
|
||||
try:
|
||||
db.session.add(new_doc)
|
||||
db.session.add(new_doc_lang)
|
||||
db.session.add(new_doc_vers)
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
error = e.args
|
||||
|
||||
# Save the file and process the document
|
||||
if error is None:
|
||||
flash('Document added successfully.')
|
||||
new_doc_vers.file_type = extension
|
||||
new_doc_vers.file_name = new_doc_vers.calc_file_name()
|
||||
new_doc_vers.file_location = new_doc_vers.calc_file_location()
|
||||
|
||||
upload_path = os.path.join(current_app.config['UPLOAD_FOLDER'], new_doc_vers.file_location)
|
||||
if not os.path.exists(upload_path):
|
||||
os.makedirs(upload_path, exist_ok=True)
|
||||
file.save(os.path.join(upload_path, new_doc_vers.file_name))
|
||||
try:
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
error = e.args
|
||||
if error is None:
|
||||
flash('Document saved successfully.')
|
||||
# TODO: processing of document to embeddings (async)
|
||||
flash('Document processing started.')
|
||||
else:
|
||||
flash('Error saving document.')
|
||||
else:
|
||||
flash('Error adding document.')
|
||||
|
||||
return render_template('document/add_document.html', form=form)
|
||||
|
||||
@@ -1,18 +1,39 @@
|
||||
from flask import current_app
|
||||
from flask_wtf import FlaskForm
|
||||
from wtforms import (StringField, PasswordField, BooleanField, SubmitField, EmailField, IntegerField, DateField,
|
||||
SelectMultipleField, FieldList, FormField)
|
||||
from wtforms.validators import DataRequired, Length, Email, NumberRange
|
||||
SelectField, SelectMultipleField, FieldList, FormField)
|
||||
from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional
|
||||
from ..models.user import User, Role
|
||||
|
||||
|
||||
class TenantForm(FlaskForm):
|
||||
name = StringField('Name', validators=[DataRequired(), Length(max=80)])
|
||||
website = StringField('Website', validators=[DataRequired(), Length(max=255)])
|
||||
license_start_date = DateField('License Start Date', id='form-control datepicker')
|
||||
license_end_date = DateField('License End Date', id='datepicker')
|
||||
# language fields
|
||||
default_language = SelectField('Default Language', choices=[], validators=[DataRequired()])
|
||||
allowed_languages = SelectMultipleField('Allowed Languages', choices=[], validators=[DataRequired()])
|
||||
# LLM fields
|
||||
default_embedding_model = SelectField('Default Embedding Model', choices=[], validators=[DataRequired()])
|
||||
allowed_embedding_models = SelectMultipleField('Allowed Embedding Models', choices=[], validators=[DataRequired()])
|
||||
default_llm_model = SelectField('Default Large Language Model', choices=[], validators=[DataRequired()])
|
||||
allowed_llm_models = SelectMultipleField('Allowed Large Language Models', choices=[], validators=[DataRequired()])
|
||||
# license fields
|
||||
license_start_date = DateField('License Start Date', id='form-control datepicker', validators=[Optional()])
|
||||
license_end_date = DateField('License End Date', id='form-control datepicker', validators=[Optional()])
|
||||
allowed_monthly_interactions = IntegerField('Allowed Monthly Interactions', validators=[NumberRange(min=0)])
|
||||
submit = SubmitField('Submit')
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(TenantForm, self).__init__(*args, **kwargs)
|
||||
# initialise language fields
|
||||
self.default_language.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
|
||||
self.allowed_languages.choices = [(lang, lang.lower()) for lang in current_app.config['SUPPORTED_LANGUAGES']]
|
||||
# initialise LLM fields
|
||||
self.default_embedding_model.choices = [(model, model) for model in current_app.config['SUPPORTED_EMBEDDINGS']]
|
||||
self.allowed_embedding_models.choices = [(model, model) for model in current_app.config['SUPPORTED_EMBEDDINGS']]
|
||||
self.default_llm_model.choices = [(model, model) for model in current_app.config['SUPPORTED_LLMS']]
|
||||
self.allowed_llm_models.choices = [(model, model) for model in current_app.config['SUPPORTED_LLMS']]
|
||||
|
||||
|
||||
class BaseUserForm(FlaskForm):
|
||||
user_name = StringField('Name', validators=[DataRequired(), Length(max=80)])
|
||||
|
||||
@@ -35,6 +35,33 @@ def tenant():
|
||||
lic_end = request.form.get('license_end_date')
|
||||
monthly = request.form.get('allowed_monthly_interactions')
|
||||
|
||||
# language fields
|
||||
default_language = request.form.get('default_language')
|
||||
allowed_languages = request.form.getlist('allowed_languages')
|
||||
|
||||
if default_language != '':
|
||||
new_tenant.default_language = default_language
|
||||
if allowed_languages != '':
|
||||
new_tenant.allowed_languages = allowed_languages
|
||||
|
||||
# LLM fields
|
||||
default_embedding = request.form.get('default_embedding')
|
||||
allowed_embeddings = request.form.getlist('allowed_embeddings')
|
||||
|
||||
if default_embedding != '':
|
||||
new_tenant.default_embedding = default_embedding
|
||||
if allowed_embeddings != '':
|
||||
new_tenant.allowed_embeddings = allowed_embeddings
|
||||
|
||||
default_llm_model = request.form.get('default_llm_model')
|
||||
allowed_llm_models = request.form.getlist('allowed_llm_models')
|
||||
|
||||
if default_llm_model != '':
|
||||
new_tenant.default_llm_model = default_llm_model
|
||||
if allowed_llm_models != '':
|
||||
new_tenant.allowed_llm_models = allowed_llm_models
|
||||
|
||||
# license data
|
||||
if lic_start != '':
|
||||
new_tenant.license_start_date = dt.strptime(lic_start, '%Y-%m-%d')
|
||||
if lic_end != '':
|
||||
@@ -74,8 +101,12 @@ def edit_tenant(tenant_id):
|
||||
# Populate the tenant with form data
|
||||
form.populate_obj(tenant)
|
||||
db.session.commit()
|
||||
flash('User updated successfully.', 'success')
|
||||
return redirect(url_for(f"user/tenant/tenant_id"))
|
||||
print(session)
|
||||
flash('Tenant updated successfully.', 'success')
|
||||
if session.get('tenant'):
|
||||
if session['tenant'].get('id') == tenant_id:
|
||||
session['tenant'] = tenant.to_dict()
|
||||
# return redirect(url_for(f"user/tenant/tenant_id"))
|
||||
|
||||
return render_template('user/edit_tenant.html', form=form, tenant_id=tenant_id)
|
||||
|
||||
@@ -180,8 +211,9 @@ def handle_tenant_selection():
|
||||
tenant_id = request.form['tenant_id']
|
||||
the_tenant = Tenant.query.get(tenant_id)
|
||||
session['tenant'] = the_tenant.to_dict()
|
||||
print("SESSION TENANT")
|
||||
print(session['tenant'])
|
||||
session['default_language'] = the_tenant.default_language
|
||||
session['default_embedding_model'] = the_tenant.default_embedding_model
|
||||
session['default_llm_model'] = the_tenant.default_llm_model
|
||||
action = request.form['action']
|
||||
|
||||
if action == 'view_users':
|
||||
|
||||
Reference in New Issue
Block a user