pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Adding functionality for listing and editing assets

Browse Source 
- Started adding functionality for creating a 'full_documents' list view.
This commit is contained in:
Josako
2025-07-03 11:14:10 +02:00
parent 51d029d960
commit 50773fe602
15 changed files with 882 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
eveai_app/views/document_views.py
View File

@@ -18,16 +18,16 @@ from common.utils.document_utils import create_document_stack, start_embedding_t
edit_document, \
edit_document_version, refresh_document, clean_url, is_file_type_supported_by_catalog
from common.utils.dynamic_field_utils import create_default_config_from_type_config
from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
EveAIDoubleURLException, EveAIException
from common.utils.eveai_exceptions import EveAIException
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, \
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
from .document_list_view import DocumentListView
from .document_version_list_view import DocumentVersionListView
from eveai_app.views.list_views.document_list_view import DocumentListView
from eveai_app.views.list_views.document_version_list_view import DocumentVersionListView
from eveai_app.views.list_views.full_document_list_view import FullDocumentListView
document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -499,6 +499,18 @@ def documents():
return view.get()
@document_bp.route('/full_documents', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
def full_documents():
catalog_id = session.get('catalog_id', None)
if not catalog_id:
flash('You need to set a Session Catalog before viewing Full Documents', 'warning')
return redirect(prefixed_url_for('document_bp.catalogs'))
view = FullDocumentListView(Document, 'document/full_documents.html', per_page=10)
return view.get()
@document_bp.route('/handle_document_selection', methods=['POST'])
@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
def handle_document_selection():
@@ -665,6 +677,59 @@ def handle_document_version_selection():
return redirect(prefixed_url_for('document_bp.document_versions', document_id=doc_vers.doc_id))
@document_bp.route('/handle_full_document_selection', methods=['POST'])
@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
def handle_full_document_selection():
selected_row = request.form['selected_row']
action = request.form['action']
try:
# Parse the selected row to get document ID (first column) and version ID (fifth column)
row_data = ast.literal_eval(selected_row)
selected_doc_id = row_data.get('value')
# We need to retrieve the corresponding row data to get the version ID
# This is a bit complex with the current structure, so we'll use a different approach
if action in ['edit_document', 'document_versions', 'refresh_document']:
# Actions that need document ID
match action:
case 'edit_document':
return redirect(prefixed_url_for('document_bp.edit_document_view', document_id=selected_doc_id))
case 'document_versions':
return redirect(prefixed_url_for('document_bp.document_versions', document_id=selected_doc_id))
case 'refresh_document':
refresh_document_view(selected_doc_id)
return redirect(prefixed_url_for('document_bp.full_documents'))
else:
# Actions that need version ID
# We need to get the version ID from the selected row in the table
# We'll extract it from the form data and the version ID is in the 5th cell (index 4)
version_id_cell = int(request.form.get('version_id', 0))
# If we couldn't get a version ID, try to find the latest version for this document
if not version_id_cell:
doc_version = DocumentVersion.query.filter_by(doc_id=selected_doc_id).order_by(desc(DocumentVersion.id)).first()
if doc_version:
version_id_cell = doc_version.id
else:
flash('No document version found for this document.', 'error')
return redirect(prefixed_url_for('document_bp.full_documents'))
match action:
case 'edit_document_version':
return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=version_id_cell))
case 'process_document_version':
process_version(version_id_cell)
return redirect(prefixed_url_for('document_bp.full_documents'))
case 'view_document_version_markdown':
return redirect(prefixed_url_for('document_bp.view_document_version_markdown', document_version_id=version_id_cell))
except (ValueError, AttributeError, KeyError) as e:
current_app.logger.error(f"Error processing full document selection: {str(e)}")
flash('Invalid selection or action. Please try again.', 'error')
return redirect(prefixed_url_for('document_bp.full_documents'))
@document_bp.route('/library_operations', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
def library_operations():

114
eveai_app/views/interaction_views.py
View File

@@ -3,9 +3,10 @@ import json
import uuid
from datetime import datetime as dt, timezone as tz
import time
from common.extensions import minio_client
from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify, url_for
from flask_security import roles_accepted
from flask_security import roles_accepted, current_user
from langchain.agents import Agent
from sqlalchemy import desc
from sqlalchemy.exc import SQLAlchemyError
@@ -775,3 +776,114 @@ def handle_specialist_magic_link_selection():
specialist_magic_link_id=specialist_ml_id))
return redirect(prefixed_url_for('interaction_bp.specialists'))
# Routes for Asset Management ---------------------------------------------------------------------
@interaction_bp.route('/assets', methods=['GET', 'POST'])
def assets():
from eveai_app.views.list_views.assets_list_view import AssetsListView
view = AssetsListView(
model=EveAIAsset,
template='interaction/assets.html',
per_page=10
)
return view.get()
@interaction_bp.route('/handle_asset_selection', methods=['POST'])
def handle_asset_selection():
action = request.form.get('action')
asset_id = request.form.get('selected_row')
current_app.logger.debug(f"Action: {action}, Asset ID: {asset_id}")
if action == 'edit_asset':
return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id))
return redirect(prefixed_url_for('interaction_bp.assets'))
@interaction_bp.route('/edit_asset/<int:asset_id>', methods=['GET', 'POST'])
def edit_asset(asset_id):
asset = EveAIAsset.query.get_or_404(asset_id)
tenant_id = session.get('tenant', {}).get('id')
if not tenant_id:
flash('Geen tenant geselecteerd', 'error')
return redirect(url_for('interaction_bp.assets'))
# Controleer of het bestandstype wordt ondersteund
if asset.file_type != 'json':
flash(
f'Bestandstype "{asset.file_type}" wordt momenteel niet ondersteund voor bewerking. Alleen JSON-bestanden kunnen worden bewerkt.',
'warning')
return redirect(url_for('interaction_bp.assets'))
if request.method == 'GET':
try:
# Haal het bestand op uit MinIO
file_data = minio_client.download_asset_file(
tenant_id,
asset.bucket_name,
asset.object_name
)
# Decodeer JSON data
json_content = json.loads(file_data.decode('utf-8'))
context = {
'asset': asset,
'json_content': json.dumps(json_content, indent=2),
'asset_id': asset_id
}
return render_template('interaction/edit_asset.html', **context)
except json.JSONDecodeError:
flash('Fout bij het laden van het JSON-bestand: ongeldig JSON-formaat', 'error')
return redirect(prefixed_url_for('interaction_bp.assets'))
except Exception as e:
current_app.logger.error(f"Error loading asset {asset_id}: {str(e)}")
flash(f'Fout bij het laden van het asset: {str(e)}', 'error')
return redirect(prefixed_url_for('interaction_bp.assets'))
elif request.method == 'POST':
try:
# Haal de bewerkte JSON data op uit het formulier
json_data = request.form.get('json_content')
if not json_data:
flash('Geen JSON data ontvangen', 'error')
return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id))
# Valideer JSON formaat
try:
parsed_json = json.loads(json_data)
except json.JSONDecodeError as e:
flash(f'Ongeldig JSON-formaat: {str(e)}', 'error')
return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id))
# Upload de bijgewerkte JSON naar MinIO
bucket_name, object_name, file_size = minio_client.upload_asset_file(
tenant_id,
asset.id,
asset.type,
asset.file_type,
json_data
)
# Update asset metadata
asset.file_size = file_size
asset.updated_at = dt.now(tz.utc)
asset.updated_by = current_user.id
asset.last_used_at = dt.now(tz.utc)
db.session.commit()
flash('Asset succesvol bijgewerkt', 'success')
return redirect(prefixed_url_for('interaction_bp.assets'))
except Exception as e:
db.session.rollback()
current_app.logger.error(f"Error saving asset {asset_id}: {str(e)}")
flash(f'Fout bij het opslaan van het asset: {str(e)}', 'error')
return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id))

0
eveai_app/views/list_views/__init__.py Normal file
View File

84
eveai_app/views/list_views/assets_list_view.py Normal file
View File

@@ -0,0 +1,84 @@
from datetime import datetime as dt, timezone as tz
from flask import request, render_template, current_app
from sqlalchemy import desc, asc
from common.models.interaction import EveAIAsset
from eveai_app.views.list_views.filtered_list_view import FilteredListView
class AssetsListView(FilteredListView):
allowed_filters = ['type', 'file_type']
allowed_sorts = ['id', 'last_used_at']
def get_query(self):
return EveAIAsset.query
def apply_filters(self, query):
filters = request.args.to_dict(flat=False)
if 'type' in filters and filters['type']:
query = query.filter(EveAIAsset.type.in_(filters['type']))
if 'file_type' in filters and filters['file_type']:
query = query.filter(EveAIAsset.file_type.in_(filters['file_type']))
return query
def apply_sorting(self, query):
sort_by = request.args.get('sort_by', 'id')
sort_order = request.args.get('sort_order', 'asc')
if sort_by in self.allowed_sorts:
column = getattr(EveAIAsset, sort_by)
if sort_order == 'asc':
query = query.order_by(asc(column))
elif sort_order == 'desc':
query = query.order_by(desc(column))
return query
def get(self):
query = self.get_query()
query = self.apply_filters(query)
query = self.apply_sorting(query)
pagination = self.paginate(query)
def format_date(date):
if isinstance(date, dt):
return date.strftime('%Y-%m-%d %H:%M:%S')
elif isinstance(date, str):
return date
else:
return ''
current_app.logger.debug(f"Assets retrieved: {pagination.items}")
rows = [
[
{'value': item.id, 'class': '', 'type': 'text'},
{'value': item.name, 'class': '', 'type': 'text'},
{'value': item.type, 'class': '', 'type': 'text'},
{'value': item.type_version, 'class': '', 'type': 'text'},
{'value': item.file_type or '', 'class': '', 'type': 'text'},
{'value': format_date(item.last_used_at), 'class': '', 'type': 'text'}
] for item in pagination.items
]
context = {
'rows': rows,
'pagination': pagination,
'filters': request.args.to_dict(flat=False),
'sort_by': request.args.get('sort_by', 'id'),
'sort_order': request.args.get('sort_order', 'asc'),
'filter_options': self.get_filter_options()
}
return render_template(self.template, **context)
def get_filter_options(self):
# Haal unieke waarden op voor filters
types = [t[0] for t in EveAIAsset.query.with_entities(EveAIAsset.type).distinct().all() if t[0]]
file_types = [f[0] for f in EveAIAsset.query.with_entities(EveAIAsset.file_type).distinct().all() if f[0]]
return {
'type': [(t, t) for t in types],
'file_type': [(f, f) for f in file_types]
}

7
eveai_app/views/document_list_view.py → eveai_app/views/list_views/document_list_view.py
View File

@@ -1,9 +1,8 @@
from datetime import datetime as dt, timezone as tz
from flask import request, render_template, session, current_app
from sqlalchemy import desc, asc, or_, and_, cast, Integer
from common.models.document import Document, Catalog
from common.utils.filtered_list_view import FilteredListView
from common.utils.view_assistants import prepare_table_for_macro
from sqlalchemy import desc, asc, or_, and_
from common.models.document import Document
from eveai_app.views.list_views.filtered_list_view import FilteredListView
class DocumentListView(FilteredListView):

4
eveai_app/views/document_version_list_view.py → eveai_app/views/list_views/document_version_list_view.py
View File

@@ -1,9 +1,9 @@
from datetime import datetime
from flask import request, render_template, session
from flask import request, render_template
from sqlalchemy import desc, asc
from common.models.document import DocumentVersion, Document
from common.utils.filtered_list_view import FilteredListView
from eveai_app.views.list_views.filtered_list_view import FilteredListView
from common.utils.view_assistants import prepare_table_for_macro

54
eveai_app/views/list_views/filtered_list_view.py Normal file
View File

@@ -0,0 +1,54 @@
from flask import request, render_template, abort
from sqlalchemy import desc, asc
class FilteredListView:
def __init__(self, model, template, per_page=10):
self.model = model
self.template = template
self.per_page = per_page
def get_query(self):
return self.model.query
def apply_filters(self, query):
filters = request.args.get('filters', {})
for key, value in filters.items():
if hasattr(self.model, key):
column = getattr(self.model, key)
if value.startswith('like:'):
query = query.filter(column.like(f"%{value[5:]}%"))
else:
query = query.filter(column == value)
return query
def apply_sorting(self, query):
sort_by = request.args.get('sort_by')
if sort_by and hasattr(self.model, sort_by):
sort_order = request.args.get('sort_order', 'asc')
column = getattr(self.model, sort_by)
if sort_order == 'desc':
query = query.order_by(desc(column))
else:
query = query.order_by(asc(column))
return query
def paginate(self, query):
page = request.args.get('page', 1, type=int)
return query.paginate(page=page, per_page=self.per_page, error_out=False)
def get(self):
query = self.get_query()
query = self.apply_filters(query)
query = self.apply_sorting(query)
pagination = self.paginate(query)
context = {
'items': pagination.items,
'pagination': pagination,
'model': self.model.__name__,
'filters': request.args.get('filters', {}),
'sort_by': request.args.get('sort_by'),
'sort_order': request.args.get('sort_order', 'asc')
}
return render_template(self.template, **context)

179
eveai_app/views/list_views/full_document_list_view.py Normal file
View File

@@ -0,0 +1,179 @@
from datetime import datetime as dt, timezone as tz
from flask import request, render_template, session, current_app
from sqlalchemy import desc, asc, or_, and_
from sqlalchemy.orm import aliased
from common.models.document import Document, DocumentVersion
from eveai_app.views.list_views.filtered_list_view import FilteredListView
from common.utils.view_assistants import prepare_table_for_macro
class FullDocumentListView(FilteredListView):
allowed_filters = ['validity', 'file_type', 'processing', 'processing_error']
allowed_sorts = ['id', 'name', 'valid_from', 'valid_to', 'file_type', 'processing_started_at',
'processing_finished_at', 'processing_error']
def __init__(self, model, template, per_page=10):
super().__init__(model, template, per_page)
self.version_alias = None
def get_query(self):
catalog_id = session.get('catalog_id')
current_app.logger.debug(f"Catalog ID: {catalog_id}")
# Fix: Selecteer alleen de id kolom in de subquery
latest_version_subquery = (
DocumentVersion.query
.with_entities(DocumentVersion.id, DocumentVersion.doc_id, DocumentVersion.url,
DocumentVersion.bucket_name, DocumentVersion.object_name,
DocumentVersion.file_type, DocumentVersion.sub_file_type,
DocumentVersion.file_size, DocumentVersion.language,
DocumentVersion.user_context, DocumentVersion.system_context,
DocumentVersion.user_metadata, DocumentVersion.system_metadata,
DocumentVersion.catalog_properties, DocumentVersion.created_at,
DocumentVersion.created_by, DocumentVersion.updated_at,
DocumentVersion.updated_by, DocumentVersion.processing,
DocumentVersion.processing_started_at, DocumentVersion.processing_finished_at,
DocumentVersion.processing_error)
.filter(DocumentVersion.id == (
DocumentVersion.query
.with_entities(DocumentVersion.id) # Selecteer alleen de id kolom
.filter(DocumentVersion.doc_id == Document.id)
.order_by(DocumentVersion.id.desc())
.limit(1)
.scalar_subquery()
))
.subquery()
)
self.version_alias = aliased(DocumentVersion, latest_version_subquery)
return Document.query.filter_by(catalog_id=catalog_id).outerjoin(
self.version_alias, Document.id == self.version_alias.doc_id
)
def apply_filters(self, query):
filters = request.args.to_dict(flat=False)
# Document filters
if 'validity' in filters:
now = dt.now(tz.utc).date()
if 'valid' in filters['validity']:
query = query.filter(
and_(
or_(Document.valid_from.is_(None), Document.valid_from <= now),
or_(Document.valid_to.is_(None), Document.valid_to >= now)
)
)
# DocumentVersion filters - use the same alias from get_query
if filters.get('file_type') and self.version_alias is not None:
query = query.filter(self.version_alias.file_type == filters['file_type'][0])
if filters.get('processing') and self.version_alias is not None:
query = query.filter(self.version_alias.processing == (filters['processing'][0] == 'true'))
if filters.get('processing_error') and self.version_alias is not None:
if filters['processing_error'][0] == 'true':
query = query.filter(self.version_alias.processing_error.isnot(None))
elif filters['processing_error'][0] == 'false':
query = query.filter(self.version_alias.processing_error.is_(None))
# Controleer of start_date een waarde heeft voordat we proberen te parsen
if filters.get('start_date') and self.version_alias is not None and filters['start_date'][0].strip():
query = query.filter(
self.version_alias.processing_started_at >= dt.strptime(filters['start_date'][0], '%Y-%m-%d'))
# Controleer of end_date een waarde heeft voordat we proberen te parsen
if filters.get('end_date') and self.version_alias is not None and filters['end_date'][0].strip():
query = query.filter(
self.version_alias.processing_finished_at <= dt.strptime(filters['end_date'][0], '%Y-%m-%d'))
return query
def apply_sorting(self, query):
sort_by = request.args.get('sort_by', 'id')
sort_order = request.args.get('sort_order', 'asc')
document_columns = ['id', 'name', 'valid_from', 'valid_to']
version_columns = ['file_type', 'processing', 'processing_started_at', 'processing_finished_at',
'processing_error']
if sort_by in self.allowed_sorts:
if sort_by in document_columns:
column = getattr(Document, sort_by)
elif sort_by in version_columns and self.version_alias is not None:
column = getattr(self.version_alias, sort_by)
else:
column = Document.id
if sort_order == 'asc':
query = query.order_by(asc(column))
elif sort_order == 'desc':
query = query.order_by(desc(column))
return query
def get(self):
query = self.get_query()
query = self.apply_filters(query)
query = self.apply_sorting(query)
pagination = self.paginate(query)
# Haal de laatste versies op voor elke document
items_with_versions = []
for doc in pagination.items:
latest_version = DocumentVersion.query.filter_by(doc_id=doc.id).order_by(desc(DocumentVersion.id)).first()
items_with_versions.append((doc, latest_version))
def format_date(date):
if isinstance(date, dt):
return date.strftime('%Y-%m-%d')
elif isinstance(date, str):
return date
else:
return ''
# Maak rijen voor de tabel met document en versie informatie
rows = []
for doc, version in items_with_versions:
if version:
row = [
{'value': doc.id, 'class': '', 'type': 'text'},
{'value': doc.name, 'class': '', 'type': 'text'},
{'value': format_date(doc.valid_from), 'class': '', 'type': 'text'},
{'value': format_date(doc.valid_to), 'class': '', 'type': 'text'},
{'value': version.id, 'class': '', 'type': 'text'},
{'value': version.file_type, 'class': '', 'type': 'text'},
{'value': 'Ja' if version.processing else 'Nee', 'class': '', 'type': 'text'},
{'value': version.processing_error or '', 'class': '', 'type': 'text'}
]
else:
row = [
{'value': doc.id, 'class': '', 'type': 'text'},
{'value': doc.name, 'class': '', 'type': 'text'},
{'value': format_date(doc.valid_from), 'class': '', 'type': 'text'},
{'value': format_date(doc.valid_to), 'class': '', 'type': 'text'},
{'value': '', 'class': '', 'type': 'text'},
{'value': '', 'class': '', 'type': 'text'},
{'value': '', 'class': '', 'type': 'text'},
{'value': '', 'class': '', 'type': 'text'}
]
rows.append(row)
context = {
'rows': rows,
'pagination': pagination,
'filters': request.args.to_dict(flat=False),
'sort_by': request.args.get('sort_by', 'id'),
'sort_order': request.args.get('sort_order', 'asc'),
'filter_options': self.get_filter_options()
}
return render_template(self.template, **context)
def get_filter_options(self):
return {
'validity': [('valid', 'Valid'), ('all', 'All')],
'file_type': [('pdf', 'PDF'), ('docx', 'DOCX')],
'processing': [('true', 'Processing'), ('false', 'Not Processing')],
'processing_error': [('true', 'With Errors'), ('false', 'Without Errors')]
}