pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removing DocumentLanguage, as both System Context and User Context are to be defined on DocumentVersion level.

Browse Source 
Finetuning of embedding workers.
This commit is contained in:
Josako
2024-06-06 15:26:49 +02:00
parent 1a25313673
commit 27b6de8734
21 changed files with 301 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
eveai_app/views/document_views.py
View File

@@ -13,9 +13,9 @@ from requests.exceptions import SSLError
from urllib.parse import urlparse
import io
from common.models.document import Document, DocumentLanguage, DocumentVersion
from common.models.document import Document, DocumentVersion
from common.extensions import db
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentLanguageForm
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
@@ -59,7 +59,7 @@ def add_document():
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
new_doc, new_doc_lang, new_doc_vers = create_document_stack(form, file, filename, extension)
new_doc, new_doc_vers = create_document_stack(form, file, filename, extension)
task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
session['tenant']['id'],
@@ -100,7 +100,7 @@ def add_url():
filename += '.html'
extension = 'html'
new_doc, new_doc_lang, new_doc_vers = create_document_stack(form, file, filename, extension)
new_doc, new_doc_vers = create_document_stack(form, file, filename, extension)
task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
session['tenant']['id'],
@@ -146,8 +146,8 @@ def handle_document_selection():
match action:
case 'edit_document':
return redirect(prefixed_url_for('document_bp.edit_document', document_id=doc_id))
case 'document_languages':
return redirect(prefixed_url_for('document_bp.document_languages', document_id=doc_id))
case 'document_versions':
return redirect(prefixed_url_for('document_bp.document_versions', document_id=doc_id))
# Add more conditions for other actions
return redirect(prefixed_url_for('document_bp.documents'))
@@ -180,81 +180,46 @@ def edit_document(document_id):
return render_template('document/edit_document.html', form=form, document_id=document_id)
@document_bp.route('/document_languages/<int:document_id>', methods=['GET', 'POST'])
@document_bp.route('/edit_document_version/<int:document_version_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def document_languages(document_id):
doc = Document.query.get_or_404(document_id)
doc_desc = f'Document {doc.id}: {doc.name}'
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = DocumentLanguage.query.filter_by(document_id=document_id).order_by(DocumentLanguage.language)
pagination = query.paginate(page=page, per_page=per_page, error_out=False)
doc_langs = pagination.items
rows = prepare_table_for_macro(doc_langs, [('id', ''), ('language', ''), ('user_context', ''),
('system_context', '')])
return render_template('document/document_languages.html', rows=rows, pagination=pagination, document=doc_desc)
@document_bp.route('/handle_document_language_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def handle_document_language_selection():
document_language_identification = request.form['selected_row']
doc_lang_id = ast.literal_eval(document_language_identification).get('value')
action = request.form['action']
match action:
case 'edit_document_language':
return redirect(prefixed_url_for('document_bp.edit_document_language', document_language_id=doc_lang_id))
case 'document_versions':
return redirect(prefixed_url_for('document_bp.document_versions', document_language_id=doc_lang_id))
# Add more conditions for other actions
return redirect(prefixed_url_for('document_bp.document_languages'))
@document_bp.route('/edit_document_language/<int:document_language_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def edit_document_language(document_language_id):
doc_lang = DocumentLanguage.query.get_or_404(document_language_id)
form = EditDocumentLanguageForm(obj=doc_lang)
def edit_document_version(document_version_id):
doc_vers = DocumentVersion.query.get_or_404(document_version_id)
form = EditDocumentVersionForm(obj=doc_vers)
if form.validate_on_submit():
doc_lang.user_context = form.user_context.data
doc_vers.user_context = form.user_context.data
update_logging_information(doc_lang, dt.now(tz.utc))
update_logging_information(doc_vers, dt.now(tz.utc))
try:
db.session.add(doc_lang)
db.session.add(doc_vers)
db.session.commit()
flash(f'Document Language {doc_lang.id} updated successfully', 'success')
flash(f'Document Version {doc_vers.id} updated successfully', 'success')
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Error updating document language: {e}', 'danger')
current_app.logger.error(f'Error updating document language {doc_lang.id} '
flash(f'Error updating document version: {e}', 'danger')
current_app.logger.error(f'Error updating document version {doc_vers.id} '
f'for tenant {session['tenant']['id']}: {e}')
else:
form_validation_failed(request, form)
return render_template('document/edit_document_language.html', form=form, document_langauge_id=document_language_id,
doc_details=f'Document {doc_lang.document.name}')
return render_template('document/edit_document_version.html', form=form, document_version_id=document_version_id,
doc_details=f'Document {doc_vers.document.name}')
@document_bp.route('/document_versions/<int:document_language_id>', methods=['GET', 'POST'])
@document_bp.route('/document_versions/<int:document_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def document_versions(document_language_id):
def document_versions(document_id):
flash(f'Processing documents is a long running process. Please be careful retriggering processing!', 'danger')
doc_lang = DocumentLanguage.query.get_or_404(document_language_id)
doc_desc = f'Document {doc_lang.document.name}, Language {doc_lang.language}'
doc_vers = DocumentVersion.query.get_or_404(document_id)
doc_desc = f'Document {doc_vers.document.name}, Language {doc_vers.language}'
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = DocumentVersion.query.filter_by(doc_lang_id=document_language_id).order_by(desc(DocumentVersion.id))
query = (DocumentVersion.query.filter_by(doc_id=document_id)
.order_by(DocumentVersion.language)
.order_by(desc(DocumentVersion.id)))
pagination = query.paginate(page=page, per_page=per_page, error_out=False)
doc_langs = pagination.items
@@ -276,6 +241,8 @@ def handle_document_version_selection():
action = request.form['action']
match action:
case 'edit_document_version':
return redirect(prefixed_url_for('document_bp.edit_document_version', document_version_id=doc_vers_id))
case 'process_document_version':
process_version(doc_vers_id)
# Add more conditions for other actions
@@ -315,17 +282,11 @@ def create_document_stack(form, file, filename, extension):
# Create the Document
new_doc = create_document(form, filename)
# Create the DocumentLanguage
new_doc_lang = create_language_for_document(new_doc, form.language.data, form.user_context.data)
# Create the DocumentVersion
new_doc_vers = DocumentVersion()
new_doc_vers.document_language = new_doc_lang
set_logging_information(new_doc_vers, dt.now(tz.utc))
new_doc_vers = create_version_for_document(new_doc, form.language.data, form.user_context.data)
try:
db.session.add(new_doc)
db.session.add(new_doc_lang)
db.session.add(new_doc_vers)
db.session.commit()
except SQLAlchemyError as e:
@@ -338,30 +299,12 @@ def create_document_stack(form, file, filename, extension):
current_app.logger.error('Unknown error')
raise
try:
new_doc_lang = db.session.merge(new_doc_lang)
new_doc_vers = db.session.merge(new_doc_vers)
new_doc_lang.latest_version_id = new_doc_vers.id
db.session.commit()
except SQLAlchemyError as e:
current_app.logger.error(f'Error adding document for tenant {session["tenant"]["id"]}: {e}')
flash('Error adding document.', 'error')
db.session.rollback()
error = e.args
raise
except Exception as e:
current_app.logger.error(f'Error adding document for tenant {session["tenant"]["id"]}: {e}')
flash('Error adding document.', 'error')
db.session.rollback()
error = e.args
raise
current_app.logger.info(f'Document added successfully for tenant {session["tenant"]["id"]}, '
f'Document Version {new_doc.id}')
upload_file_for_version(new_doc_vers, file, extension)
return new_doc, new_doc_lang, new_doc_vers
return new_doc, new_doc_vers
def log_session_state(session, msg=""):
@@ -386,21 +329,21 @@ def create_document(form, filename):
return new_doc
def create_language_for_document(document, language, user_context):
new_doc_lang = DocumentLanguage()
def create_version_for_document(document, language, user_context):
new_doc_vers = DocumentVersion()
if language == '':
new_doc_lang.language = session['default_language']
new_doc_vers.language = session['default_language']
else:
new_doc_lang.language = language
new_doc_vers.language = language
if user_context != '':
new_doc_lang.user_context = user_context
new_doc_vers.user_context = user_context
new_doc_lang.document = document
new_doc_vers.document = document
set_logging_information(new_doc_lang, dt.now(tz.utc))
set_logging_information(new_doc_vers, dt.now(tz.utc))
return new_doc_lang
return new_doc_vers
def upload_file_for_version(doc_vers, file, extension):