pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improvements to Document Interface and correcting embedding workers

Browse Source
This commit is contained in:
Josako
2024-06-04 14:59:38 +02:00
parent c660c35de4
commit 61e1372dc8
15 changed files with 486 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
eveai_app/__init__.py
View File

@@ -70,7 +70,7 @@ def create_app(config_file=None):
security_logger.setLevel(logging.DEBUG)
sqlalchemy_logger = logging.getLogger('sqlalchemy.engine')
sqlalchemy_logger.setLevel(logging.DEBUG)
log_request_middleware(app) # Add this when debugging nginx or another proxy
# log_request_middleware(app) # Add this when debugging nginx or another proxy
# Some generic Error Handling Routines
@app.errorhandler(Exception)

24
eveai_app/templates/document/document_languages.html Normal file
View File

@@ -0,0 +1,24 @@
{% extends 'base.html' %}
{% from 'macros.html' import render_selectable_table, render_pagination %}
{% block title %}Document Languages{% endblock %}
{% block content_title %}Document Languages{% endblock %}
{% block content_description %}View Languages for {{ document }}{% endblock %}
{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto">{% endblock %}
{% block content %}
<div class="container">
<form method="POST" action="{{ url_for('document_bp.handle_document_language_selection') }}">
{{ render_selectable_table(headers=["Document Language ID", "Language", "User Context", "System Context"], rows=rows, selectable=True, id="documentsTable") }}
<div class="form-group mt-3">
<button type="submit" name="action" value="edit_document_language" class="btn btn-primary">Edit Document Language</button>
<button type="submit" name="action" value="document_versions" class="btn btn-secondary">Show Document Versions</button>
</div>
</form>
</div>
{% endblock %}
{% block content_footer %}
{{ render_pagination(pagination, 'document_bp.documents') }}
{% endblock %}

23
eveai_app/templates/document/document_versions.html Normal file
View File

@@ -0,0 +1,23 @@
{% extends 'base.html' %}
{% from 'macros.html' import render_selectable_table, render_pagination %}
{% block title %}Document Versions{% endblock %}
{% block content_title %}Document Versions{% endblock %}
{% block content_description %}View Versions for {{ document }}{% endblock %}
{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto">{% endblock %}
{% block content %}
<div class="container">
<form method="POST" action="{{ url_for('document_bp.handle_document_version_selection') }}">
{{ render_selectable_table(headers=["Document Version ID", "URL", "File Location", "File Name", "File Type", "Processing", "Processing Start", "Proceeing Finish"], rows=rows, selectable=True, id="versionsTable") }}
<div class="form-group mt-3">
<button type="submit" name="action" value="process_document_version" class="btn btn-primary">Process Document Version</button>
</div>
</form>
</div>
{% endblock %}
{% block content_footer %}
{{ render_pagination(pagination, 'document_bp.documents') }}
{% endblock %}

11
eveai_app/templates/document/documents.html
View File

@@ -1,5 +1,5 @@
{% extends 'base.html' %}
{% from 'macros.html' import render_nested_table, render_pagination %}
{% from 'macros.html' import render_selectable_table, render_pagination %}
{% block title %}Documents{% endblock %}
@@ -9,8 +9,13 @@
{% block content %}
<div class="container">
<!-- Documents Table -->
{{ render_nested_table(headers=["Name", "Created At", "Valid From", "Languages & Versions"], rows=rows) }}
<form method="POST" action="{{ url_for('document_bp.handle_document_selection') }}">
{{ render_selectable_table(headers=["Document ID", "Name", "Valid From", "Valid To"], rows=rows, selectable=True, id="documentsTable") }}
<div class="form-group mt-3">
<button type="submit" name="action" value="edit_document" class="btn btn-primary">Edit Document</button>
<button type="submit" name="action" value="document_languages" class="btn btn-secondary">Show Document Languages</button>
</div>
</form>
</div>
{% endblock %}

18
eveai_app/templates/document/edit_document.html Normal file
View File

@@ -0,0 +1,18 @@
{% extends "base.html" %}
{% from "macros.html" import render_field %}
{% block title %}Update Document{% endblock %}
{% block content_title %}Update Document{% endblock %}
{% block content_description %}Update document details.{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Update Document</button>
</form>
{% endblock %}

18
eveai_app/templates/document/edit_document_language.html Normal file
View File

@@ -0,0 +1,18 @@
{% extends "base.html" %}
{% from "macros.html" import render_field %}
{% block title %}Update Document Language{% endblock %}
{% block content_title %}Update Document Language{% endblock %}
{% block content_description %}Update document language for {{ doc_details }}.{% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = ['language', 'system_context'] %}
{% set exclude_fields = [] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Update Document</button>
</form>
{% endblock %}

51
eveai_app/templates/macros.html
View File

@@ -217,57 +217,6 @@
</div>
{% endmacro %}
{% macro render_seamless_table(headers, rows) %}
{% macro render_integrated_table(headers, data) %}
<div class="table-responsive">
<table class="table align-items-center mb-0">
<thead>
<tr>
{% for header in headers %}
<th class="text-uppercase text-secondary text-xxs font-weight-bolder opacity-7">{{ header }}</th>
{% endfor %}
</tr>
</thead>
<tbody>
{% for entry in data %}
{% if entry.is_group and entry.sub_rows %}
{% for sub_row in entry.sub_rows %}
<tr>
{% for cell in sub_row %}
{% if cell %}
<td class="{{ cell.class }}">
{% if cell.type == 'text' %}
<p class="text-xs {{ cell.text_class }}">{{ cell.value }}</p>
{% else %}
{{ cell.value }}
{% endif %}
</td>
{% else %}
<td></td>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
{% else %}
<tr>
{% for cell in entry %}
<td class="{{ cell.class }}">
{% if cell.type == 'text' %}
<p class="text-xs {{ cell.text_class }}">{{ cell.value }}</p>
{% else %}
{{ cell.value }}
{% endif %}
</td>
{% endfor %}
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</div>
{% endmacro %}
{% endmacro %}
{% macro render_pagination(pagination, endpoint) %}
<nav aria-label="Page navigation">
<ul class="pagination">

29
eveai_app/templates/user/select_tenant.html
View File

@@ -20,32 +20,3 @@
{{ render_pagination(pagination, 'user_bp.select_tenant') }}
{% endblock %}
{#{% block scripts %}#}
{#<script>#}
{#$(document).ready(function() {#}
{# $('#tenantsTable').DataTable({#}
{# 'columnDefs': [#}
{# {#}
{# 'targets': 0,#}
{# 'searchable': false,#}
{# 'orderable': false,#}
{# 'className': 'dt-body-center',#}
{# },#}
{# {#}
{# 'targets': 1,#}
{# 'orderable': true#}
{# },#}
{# {#}
{# 'targets': 2,#}
{# 'orderable': true#}
{# },#}
{# {#}
{# 'targets': 2,#}
{# 'orderable': true#}
{# },#}
{# ],#}
{# 'order': [[1, 'asc']]#}
{# });#}
{#});#}
{#</script>#}
{#{% endblock %}#}

19
eveai_app/views/document_forms.py
View File

@@ -37,3 +37,22 @@ class AddURLForm(FlaskForm):
self.language.choices = [(language, language) for language in
session.get('tenant').get('allowed_languages')]
self.language.data = session.get('default_language')
class EditDocumentForm(FlaskForm):
name = StringField('Name', validators=[Length(max=100)])
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
valid_to = DateField('Valid to', id='form-control datepicker', validators=[Optional()])
submit = SubmitField('Submit')
class EditDocumentLanguageForm(FlaskForm):
language = StringField('Language')
user_context = TextAreaField('User Context', validators=[Optional()])
system_context = TextAreaField('System Context', validators=[Optional()])
submit = SubmitField('Submit')

225
eveai_app/views/document_views.py
View File

@@ -1,3 +1,4 @@
import ast
import os
from datetime import datetime as dt, timezone as tz
from flask import request, redirect, flash, render_template, Blueprint, session, current_app
@@ -14,10 +15,11 @@ import io
from common.models.document import Document, DocumentLanguage, DocumentVersion
from common.extensions import db
from .document_forms import AddDocumentForm, AddURLForm
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentLanguageForm
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -29,13 +31,20 @@ def log_before_request():
@document_bp.after_request
def log_after_request(response):
current_app.logger.debug(f"After request (document_bp): {request.method} {request.url} - Status: {response.status}")
current_app.logger.debug(
f"After request (document_bp): {request.method} {request.url} - Status: {response.status}")
return response
@document_bp.before_request
def before_request():
mw_before_request()
try:
mw_before_request()
except Exception as e:
current_app.logger.error(f'Error switching schema in Document Blueprint: {e}')
for role in current_user.roles:
current_app.logger.debug(f'User {current_user.email} has role {role.name}')
raise
@document_bp.route('/add_document', methods=['GET', 'POST'])
@@ -44,7 +53,7 @@ def add_document():
form = AddDocumentForm()
# If the form is submitted
if request.method == 'POST' and form.validate_on_submit():
if form.validate_on_submit():
current_app.logger.info(f'Adding document for tenant {session["tenant"]["id"]}')
file = form.file.data
filename = secure_filename(file.filename)
@@ -59,8 +68,12 @@ def add_document():
current_app.logger.info(f'Embedding creation started for tenant {session["tenant"]["id"]}, '
f'Document Version {new_doc_vers.id}. '
f'Embedding creation task: {task.id}')
flash(f'Processing on document {new_doc.name}, version {new_doc_vers.id} started. Task ID: {task.id}.',
'success')
return redirect(prefixed_url_for('document_bp.documents'))
else:
form_validation_failed(request, form)
return render_template('document/add_document.html', form=form)
@@ -71,7 +84,7 @@ def add_url():
form = AddURLForm()
# If the form is submitted
if request.method == 'POST' and form.validate_on_submit():
if form.validate_on_submit():
current_app.logger.info(f'Adding document for tenant {session["tenant"]["id"]}')
url = form.url.data
@@ -96,8 +109,12 @@ def add_url():
current_app.logger.info(f'Embedding creation started for tenant {session["tenant"]["id"]}, '
f'Document Version {new_doc_vers.id}. '
f'Embedding creation task: {task.id}')
flash(f'Processing on document {new_doc.name}, version {new_doc_vers.id} started. Task ID: {task.id}.',
'success')
return redirect(prefixed_url_for('document_bp.documents'))
else:
form_validation_failed(request, form)
return render_template('document/add_url.html', form=form)
@@ -108,25 +125,178 @@ def documents():
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = Document.query.order_by(desc(Document.created_at)).options(
joinedload(Document.languages).joinedload(DocumentLanguage.versions))
query = Document.query.order_by(desc(Document.created_at))
pagination = query.paginate(page=page, per_page=per_page, error_out=False)
docs = pagination.items
rows = prepare_document_data(docs)
rows = prepare_table_for_macro(docs, [('id', ''), ('name', ''), ('valid_from', ''), ('valid_to', '')])
return render_template('document/documents.html', rows=rows, pagination=pagination)
@document_bp.route('/process_version/<int:version_id>', methods=['POST'])
@document_bp.route('/handle_document_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def process_version(version_id):
version = DocumentVersion.query.get_or_404(version_id)
if not version.processing:
print(f'Placeholder for processing version: {version_id}')
def handle_document_selection():
document_identification = request.form['selected_row']
doc_id = ast.literal_eval(document_identification).get('value')
return redirect(prefixed_url_for('documents'))
action = request.form['action']
match action:
case 'edit_document':
return redirect(prefixed_url_for('document_bp.edit_document', document_id=doc_id))
case 'document_languages':
return redirect(prefixed_url_for('document_bp.document_languages', document_id=doc_id))
# Add more conditions for other actions
return redirect(prefixed_url_for('document_bp.documents'))
@document_bp.route('/edit_document/<int:document_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def edit_document(document_id):
doc = Document.query.get_or_404(document_id)
form = EditDocumentForm(obj=doc)
if form.validate_on_submit():
doc.name = form.name.data
doc.valid_from = form.valid_from.data
doc.valid_to = form.valid_to.data
update_logging_information(doc, dt.now(tz.utc))
try:
db.session.add(doc)
db.session.commit()
flash(f'Document {doc.id} updated successfully', 'success')
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Error updating document: {e}', 'danger')
current_app.logger.error(f'Error updating document: {e}')
else:
form_validation_failed(request, form)
return render_template('document/edit_document.html', form=form, document_id=document_id)
@document_bp.route('/document_languages/<int:document_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def document_languages(document_id):
doc = Document.query.get_or_404(document_id)
doc_desc = f'Document {doc.id}: {doc.name}'
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = DocumentLanguage.query.filter_by(document_id=document_id).order_by(DocumentLanguage.language)
pagination = query.paginate(page=page, per_page=per_page, error_out=False)
doc_langs = pagination.items
rows = prepare_table_for_macro(doc_langs, [('id', ''), ('language', ''), ('user_context', ''),
('system_context', '')])
return render_template('document/document_languages.html', rows=rows, pagination=pagination, document=doc_desc)
@document_bp.route('/handle_document_language_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def handle_document_language_selection():
document_language_identification = request.form['selected_row']
doc_lang_id = ast.literal_eval(document_language_identification).get('value')
action = request.form['action']
match action:
case 'edit_document_language':
return redirect(prefixed_url_for('document_bp.edit_document_language', document_language_id=doc_lang_id))
case 'document_versions':
return redirect(prefixed_url_for('document_bp.document_versions', document_language_id=doc_lang_id))
# Add more conditions for other actions
return redirect(prefixed_url_for('document_bp.document_languages'))
@document_bp.route('/edit_document_language/<int:document_language_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def edit_document_language(document_language_id):
doc_lang = DocumentLanguage.query.get_or_404(document_language_id)
form = EditDocumentLanguageForm(obj=doc_lang)
if form.validate_on_submit():
doc_lang.user_context = form.user_context.data
update_logging_information(doc_lang, dt.now(tz.utc))
try:
db.session.add(doc_lang)
db.session.commit()
flash(f'Document Language {doc_lang.id} updated successfully', 'success')
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Error updating document language: {e}', 'danger')
current_app.logger.error(f'Error updating document language {doc_lang.id} '
f'for tenant {session['tenant']['id']}: {e}')
else:
form_validation_failed(request, form)
return render_template('document/edit_document_language.html', form=form, document_langauge_id=document_language_id,
doc_details=f'Document {doc_lang.document.name}')
@document_bp.route('/document_versions/<int:document_language_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def document_versions(document_language_id):
flash(f'Processing documents is a long running process. Please be careful retriggering processing!', 'danger')
doc_lang = DocumentLanguage.query.get_or_404(document_language_id)
doc_desc = f'Document {doc_lang.document.name}, Language {doc_lang.language}'
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = DocumentVersion.query.filter_by(doc_lang_id=document_language_id).order_by(desc(DocumentVersion.id))
pagination = query.paginate(page=page, per_page=per_page, error_out=False)
doc_langs = pagination.items
rows = prepare_table_for_macro(doc_langs, [('id', ''), ('url', ''), ('file_location', ''),
('file_name', ''), ('file_type', ''),
('processing', ''), ('processing_started_at', ''),
('processing_finished_at', '')])
return render_template('document/document_versions.html', rows=rows, pagination=pagination, document=doc_desc)
@document_bp.route('/handle_document_version_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def handle_document_version_selection():
document_version_identification = request.form['selected_row']
doc_vers_id = ast.literal_eval(document_version_identification).get('value')
action = request.form['action']
match action:
case 'process_document_version':
process_version(doc_vers_id)
# Add more conditions for other actions
doc_vers = DocumentVersion.query.get_or_404(doc_vers_id)
return redirect(prefixed_url_for('document_bp.document_versions', document_language_id=doc_vers.doc_lang_id))
def process_version(version_id):
task = current_celery.send_task('create_embeddings', queue='embeddings', args=[
session['tenant']['id'],
version_id,
])
current_app.logger.info(f'Embedding creation retriggered by user {current_user.id}, {current_user.email} '
f'for tenant {session["tenant"]["id"]}, '
f'Document Version {version_id}. '
f'Embedding creation task: {task.id}')
flash(f'Processing for document version {version_id} retriggered successfully...', 'success')
return redirect(prefixed_url_for('document_bp.documents'))
def set_logging_information(obj, timestamp):
@@ -136,6 +306,11 @@ def set_logging_information(obj, timestamp):
obj.updated_by = current_user.id
def update_logging_information(obj, timestamp):
obj.updated_at = timestamp
obj.updated_by = current_user.id
def create_document_stack(form, file, filename, extension):
# Create the Document
new_doc = create_document(form, filename)
@@ -155,7 +330,7 @@ def create_document_stack(form, file, filename, extension):
db.session.commit()
except SQLAlchemyError as e:
current_app.logger.error(f'Error adding document for tenant {session["tenant"]["id"]}: {e}')
flash('Error adding document.', 'error')
flash('Error adding document.', 'alert')
db.session.rollback()
error = e.args
raise
@@ -277,25 +452,6 @@ def fetch_html(url):
return response.text
# Sample code for adding or updating versions and ensuring latest_version is set in DocumentLanguage
# def add_or_update_version(language_id, version_data):
# new_version = Version(language_id=language_id, **version_data)
# db.session.add(new_version)
# db.session.flush() # Ensures new_version gets an ID assigned if it's new
#
# # Assuming we always call this when we know it's the latest
# language = Language.query.get(language_id)
# language.latest_version_id = new_version.id
# db.session.commit()
# sample code for using latest_version in the application
# @app.route('/language/<int:language_id>')
# def show_language(language_id):
# language = Language.query.get_or_404(language_id)
# latest_version = language.latest_version # This is now a direct, efficient database access
# return render_template('language_details.html', language=language, latest_version=latest_version)
def prepare_document_data(docs):
rows = []
for doc in docs:
@@ -340,4 +496,3 @@ def prepare_document_data(docs):
'sub_rows': languages_rows})
rows.append(doc_row)
return rows