diff --git a/common/utils/document_utils.py b/common/utils/document_utils.py index 79d02b1..d0b0c90 100644 --- a/common/utils/document_utils.py +++ b/common/utils/document_utils.py @@ -15,12 +15,11 @@ from config.type_defs.processor_types import PROCESSOR_TYPES from .config_field_types import normalize_json_field from .eveai_exceptions import (EveAIInvalidLanguageException, EveAIDoubleURLException, EveAIUnsupportedFileType, EveAIInvalidCatalog, EveAIInvalidDocument, EveAIInvalidDocumentVersion, EveAIException) +from .minio_utils import MIB_CONVERTOR from ..models.user import Tenant from common.utils.model_logging_utils import set_logging_information, update_logging_information from common.services.entitlements import LicenseUsageServices -MB_CONVERTOR = 1_048_576 - def get_file_size(file): try: @@ -39,7 +38,7 @@ def get_file_size(file): def create_document_stack(api_input, file, filename, extension, tenant_id): # Precheck if we can add a document to the stack - LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file)/MB_CONVERTOR) + LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file) / MIB_CONVERTOR) # Create the Document catalog_id = int(api_input.get('catalog_id')) @@ -144,7 +143,7 @@ def upload_file_for_version(doc_vers, file, extension, tenant_id): ) doc_vers.bucket_name = bn doc_vers.object_name = on - doc_vers.file_size = size / MB_CONVERTOR # Convert bytes to MB + doc_vers.file_size = size / MIB_CONVERTOR # Convert bytes to MB db.session.commit() current_app.logger.info(f'Successfully saved document to MinIO for tenant {tenant_id} for ' @@ -354,7 +353,7 @@ def refresh_document_with_content(doc_id: int, tenant_id: int, file_content: byt old_doc_vers = DocumentVersion.query.filter_by(doc_id=doc_id).order_by(desc(DocumentVersion.id)).first() # Precheck if we have enough quota for the new version - LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file_content) / MB_CONVERTOR) + LicenseUsageServices.check_storage_and_embedding_quota(tenant_id, get_file_size(file_content) / MIB_CONVERTOR) # Create new version with same file type as original extension = old_doc_vers.file_type diff --git a/common/utils/minio_utils.py b/common/utils/minio_utils.py index 78ca4d1..c5e2a75 100644 --- a/common/utils/minio_utils.py +++ b/common/utils/minio_utils.py @@ -4,6 +4,9 @@ from flask import Flask import io from werkzeug.datastructures import FileStorage +MIB_CONVERTOR = 1_048_576 + + class MinioClient: def __init__(self): self.client = None @@ -58,7 +61,7 @@ class MinioClient: raise Exception(f"Error occurred while uploading file: {err}") def upload_asset_file(self, tenant_id: int, asset_id: int, asset_type: str, file_type: str, - file_data: bytes | FileStorage | io.BytesIO | str,) -> tuple[str, str, int]: + file_data: bytes | FileStorage | io.BytesIO | str, ) -> tuple[str, str, int]: bucket_name = self.generate_bucket_name(tenant_id) object_name = self.generate_asset_name(asset_id, asset_type, file_type) @@ -119,4 +122,11 @@ class MinioClient: try: self.client.remove_object(bucket_name, object_name) except S3Error as err: - raise Exception(f"Error occurred while deleting object: {err}") \ No newline at end of file + raise Exception(f"Error occurred while deleting object: {err}") + + def get_bucket_size(self, tenant_id: int) -> int: + bucket_name = self.generate_bucket_name(tenant_id) + total_size = 0 + for obj in self.client.list_objects(bucket_name, recursive=True): + total_size += obj.size + return total_size diff --git a/eveai_app/templates/document/full_documents.html b/eveai_app/templates/document/full_documents.html new file mode 100644 index 0000000..584eafb --- /dev/null +++ b/eveai_app/templates/document/full_documents.html @@ -0,0 +1,114 @@ +{% extends 'base.html' %} +{% from 'macros.html' import render_selectable_table, render_pagination, render_filter_field, render_date_filter_field, render_collapsible_section, render_selectable_sortable_table_with_dict_headers %} + +{% block title %}Complete Document Overview{% endblock %} + +{% block content_title %}Complete Document Overview{% endblock %} +{% block content_description %}View Documents with Latest Version for Catalog {% if session.catalog_name %}{{ session.catalog_name }}{% else %}No Catalog{% endif %}{% endblock %} +{% block content_class %}
{% endblock %} + +{% block content %} + + {% set filter_form %} +
+ {{ render_filter_field('validity', 'Validity', filter_options['validity'], filters.get('validity', [])) }} + {{ render_filter_field('file_type', 'File Type', filter_options['file_type'], filters.get('file_type', [])) }} + {{ render_filter_field('processing', 'Processing Status', filter_options['processing'], filters.get('processing', [])) }} + {{ render_filter_field('processing_error', 'Error Status', filter_options['processing_error'], filters.get('processing_error', [])) }} + {{ render_date_filter_field('start_date', 'Processing Start Date', filters.get('start_date', [])) }} + {{ render_date_filter_field('end_date', 'Processing End Date', filters.get('end_date', [])) }} + + +
+ {% endset %} + + {{ render_collapsible_section('Filter', 'Filter Options', filter_form) }} + +
+
+ + + + + {{ render_selectable_sortable_table_with_dict_headers( + headers=[ + {"text": "Document ID", "sort": "id"}, + {"text": "Name", "sort": "name"}, + {"text": "Valid From", "sort": "valid_from"}, + {"text": "Valid To", "sort": "valid_to"}, + {"text": "Version ID", "sort": ""}, + {"text": "File Type", "sort": "file_type"}, + {"text": "Processing", "sort": "processing"}, + {"text": "Error", "sort": "processing_error"} + ], + rows=rows, + selectable=True, + id="fullDocumentsTable", + sort_by=sort_by, + sort_order=sort_order + ) }} +
+
+ + + + + + +
+
+
+
+{% endblock %} + +{% block content_footer %} + {{ render_pagination(pagination, 'document_bp.full_documents') }} +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/eveai_app/templates/interaction/assets.html b/eveai_app/templates/interaction/assets.html new file mode 100644 index 0000000..1bbf72a --- /dev/null +++ b/eveai_app/templates/interaction/assets.html @@ -0,0 +1,97 @@ + +{% extends 'base.html' %} +{% from 'macros.html' import render_selectable_table, render_pagination, render_filter_field, render_date_filter_field, render_collapsible_section, render_selectable_sortable_table_with_dict_headers %} + +{% block title %}Assets{% endblock %} + +{% block content_title %}Assets{% endblock %} +{% block content_description %}View Assets{% endblock %} +{% block content_class %}
{% endblock %} + +{% block content %} + + {% set filter_form %} +
+ {{ render_filter_field('type', 'Type', filter_options['type'], filters.get('type', [])) }} + {{ render_filter_field('file_type', 'Bestandstype', filter_options['file_type'], filters.get('file_type', [])) }} + + +
+ {% endset %} + + {{ render_collapsible_section('Filter', 'Filter Options', filter_form) }} + +
+
+ + {{ render_selectable_sortable_table_with_dict_headers( + headers=[ + {"text": "ID", "sort": "id"}, + {"text": "Naam", "sort": "name"}, + {"text": "Type", "sort": "type"}, + {"text": "Type Versie", "sort": "type_version"}, + {"text": "Bestandstype", "sort": "file_type"}, + {"text": "Laatst Gebruikt", "sort": "last_used_at"} + ], + rows=rows, + selectable=True, + id="assetsTable", + sort_by=sort_by, + sort_order=sort_order + ) }} +
+
+ +
+
+
+
+{% endblock %} + +{% block content_footer %} + {{ render_pagination(pagination, 'interaction_bp.assets') }} +{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/interaction/edit_asset.html b/eveai_app/templates/interaction/edit_asset.html new file mode 100644 index 0000000..d219df4 --- /dev/null +++ b/eveai_app/templates/interaction/edit_asset.html @@ -0,0 +1,201 @@ +{% extends 'base.html' %} + +{% block title %}Edit Asset{% endblock %} + +{% block content_title %}Edit Asset{% endblock %} +{% block content_description %}Edit Asset: {{ asset.name }}{% endblock %} +{% block content_class %}
{% endblock %} + +{% block content %} +
+
+
+
+

Asset Details

+
+
+
+
+ +
+
+

ID: {{ asset.id }}

+

Name: {{ asset.name }}

+

Type: {{ asset.type }}

+
+
+

Type Version: {{ asset.type_version }}

+

File Type: {{ asset.file_type }}

+

File Size: {{ asset.file_size or 'N/A' }} bytes

+
+
+ +
+ + +
+
+
+
JSON Content
+ + +
+ +
+
+
+
+ +
+ + +
+
+
+ +
+ +
+
+
+
+
+
+
+{% endblock %} + + +{% block scripts %} +{{ super() }} + + +{% endblock %} \ No newline at end of file diff --git a/eveai_app/templates/navbar.html b/eveai_app/templates/navbar.html index 4f7a657..87a226a 100644 --- a/eveai_app/templates/navbar.html +++ b/eveai_app/templates/navbar.html @@ -100,6 +100,7 @@ {'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Add URL', 'url': '/document/add_url', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Documents', 'url': '/document/documents', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, + {'name': 'Full Documents', 'url': '/document/full_documents', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Document Versions', 'url': '/document/document_versions_list', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Library Operations', 'url': '/document/library_operations', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, ]) }} @@ -108,6 +109,7 @@ {{ dropdown('Interactions', 'hub', [ {'name': 'Specialists', 'url': '/interaction/specialists', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Specialist Magic Links', 'url': '/interaction/specialist_magic_links', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, + {'name': 'Assets', 'url': '/interaction/assets', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Chat Sessions', 'url': '/interaction/chat_sessions', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, ]) }} {% endif %} diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index 410cba2..0e2e425 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -18,16 +18,16 @@ from common.utils.document_utils import create_document_stack, start_embedding_t edit_document, \ edit_document_version, refresh_document, clean_url, is_file_type_supported_by_catalog from common.utils.dynamic_field_utils import create_default_config_from_type_config -from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \ - EveAIDoubleURLException, EveAIException +from common.utils.eveai_exceptions import EveAIException from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, \ CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm from common.utils.middleware import mw_before_request from common.utils.celery_utils import current_celery from common.utils.nginx_utils import prefixed_url_for from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro -from .document_list_view import DocumentListView -from .document_version_list_view import DocumentVersionListView +from eveai_app.views.list_views.document_list_view import DocumentListView +from eveai_app.views.list_views.document_version_list_view import DocumentVersionListView +from eveai_app.views.list_views.full_document_list_view import FullDocumentListView document_bp = Blueprint('document_bp', __name__, url_prefix='/document') @@ -499,6 +499,18 @@ def documents(): return view.get() +@document_bp.route('/full_documents', methods=['GET', 'POST']) +@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') +def full_documents(): + catalog_id = session.get('catalog_id', None) + if not catalog_id: + flash('You need to set a Session Catalog before viewing Full Documents', 'warning') + return redirect(prefixed_url_for('document_bp.catalogs')) + + view = FullDocumentListView(Document, 'document/full_documents.html', per_page=10) + return view.get() + + @document_bp.route('/handle_document_selection', methods=['POST']) @roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') def handle_document_selection(): @@ -665,6 +677,59 @@ def handle_document_version_selection(): return redirect(prefixed_url_for('document_bp.document_versions', document_id=doc_vers.doc_id)) +@document_bp.route('/handle_full_document_selection', methods=['POST']) +@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') +def handle_full_document_selection(): + selected_row = request.form['selected_row'] + action = request.form['action'] + + try: + # Parse the selected row to get document ID (first column) and version ID (fifth column) + row_data = ast.literal_eval(selected_row) + selected_doc_id = row_data.get('value') + + # We need to retrieve the corresponding row data to get the version ID + # This is a bit complex with the current structure, so we'll use a different approach + if action in ['edit_document', 'document_versions', 'refresh_document']: + # Actions that need document ID + match action: + case 'edit_document': + return redirect(prefixed_url_for('document_bp.edit_document_view', document_id=selected_doc_id)) + case 'document_versions': + return redirect(prefixed_url_for('document_bp.document_versions', document_id=selected_doc_id)) + case 'refresh_document': + refresh_document_view(selected_doc_id) + return redirect(prefixed_url_for('document_bp.full_documents')) + else: + # Actions that need version ID + # We need to get the version ID from the selected row in the table + # We'll extract it from the form data and the version ID is in the 5th cell (index 4) + version_id_cell = int(request.form.get('version_id', 0)) + + # If we couldn't get a version ID, try to find the latest version for this document + if not version_id_cell: + doc_version = DocumentVersion.query.filter_by(doc_id=selected_doc_id).order_by(desc(DocumentVersion.id)).first() + if doc_version: + version_id_cell = doc_version.id + else: + flash('No document version found for this document.', 'error') + return redirect(prefixed_url_for('document_bp.full_documents')) + + match action: + case 'edit_document_version': + return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=version_id_cell)) + case 'process_document_version': + process_version(version_id_cell) + return redirect(prefixed_url_for('document_bp.full_documents')) + case 'view_document_version_markdown': + return redirect(prefixed_url_for('document_bp.view_document_version_markdown', document_version_id=version_id_cell)) + except (ValueError, AttributeError, KeyError) as e: + current_app.logger.error(f"Error processing full document selection: {str(e)}") + flash('Invalid selection or action. Please try again.', 'error') + + return redirect(prefixed_url_for('document_bp.full_documents')) + + @document_bp.route('/library_operations', methods=['GET', 'POST']) @roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') def library_operations(): diff --git a/eveai_app/views/interaction_views.py b/eveai_app/views/interaction_views.py index 21e3bba..295806d 100644 --- a/eveai_app/views/interaction_views.py +++ b/eveai_app/views/interaction_views.py @@ -3,9 +3,10 @@ import json import uuid from datetime import datetime as dt, timezone as tz import time +from common.extensions import minio_client from flask import request, redirect, flash, render_template, Blueprint, session, current_app, jsonify, url_for -from flask_security import roles_accepted +from flask_security import roles_accepted, current_user from langchain.agents import Agent from sqlalchemy import desc from sqlalchemy.exc import SQLAlchemyError @@ -775,3 +776,114 @@ def handle_specialist_magic_link_selection(): specialist_magic_link_id=specialist_ml_id)) return redirect(prefixed_url_for('interaction_bp.specialists')) + + +# Routes for Asset Management --------------------------------------------------------------------- +@interaction_bp.route('/assets', methods=['GET', 'POST']) +def assets(): + from eveai_app.views.list_views.assets_list_view import AssetsListView + view = AssetsListView( + model=EveAIAsset, + template='interaction/assets.html', + per_page=10 + ) + return view.get() + + +@interaction_bp.route('/handle_asset_selection', methods=['POST']) +def handle_asset_selection(): + action = request.form.get('action') + asset_id = request.form.get('selected_row') + current_app.logger.debug(f"Action: {action}, Asset ID: {asset_id}") + + if action == 'edit_asset': + return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id)) + + return redirect(prefixed_url_for('interaction_bp.assets')) + + +@interaction_bp.route('/edit_asset/', methods=['GET', 'POST']) +def edit_asset(asset_id): + asset = EveAIAsset.query.get_or_404(asset_id) + tenant_id = session.get('tenant', {}).get('id') + + if not tenant_id: + flash('Geen tenant geselecteerd', 'error') + return redirect(url_for('interaction_bp.assets')) + + # Controleer of het bestandstype wordt ondersteund + if asset.file_type != 'json': + flash( + f'Bestandstype "{asset.file_type}" wordt momenteel niet ondersteund voor bewerking. Alleen JSON-bestanden kunnen worden bewerkt.', + 'warning') + return redirect(url_for('interaction_bp.assets')) + + if request.method == 'GET': + try: + # Haal het bestand op uit MinIO + file_data = minio_client.download_asset_file( + tenant_id, + asset.bucket_name, + asset.object_name + ) + + # Decodeer JSON data + json_content = json.loads(file_data.decode('utf-8')) + + context = { + 'asset': asset, + 'json_content': json.dumps(json_content, indent=2), + 'asset_id': asset_id + } + + return render_template('interaction/edit_asset.html', **context) + + except json.JSONDecodeError: + flash('Fout bij het laden van het JSON-bestand: ongeldig JSON-formaat', 'error') + return redirect(prefixed_url_for('interaction_bp.assets')) + except Exception as e: + current_app.logger.error(f"Error loading asset {asset_id}: {str(e)}") + flash(f'Fout bij het laden van het asset: {str(e)}', 'error') + return redirect(prefixed_url_for('interaction_bp.assets')) + + elif request.method == 'POST': + try: + # Haal de bewerkte JSON data op uit het formulier + json_data = request.form.get('json_content') + + if not json_data: + flash('Geen JSON data ontvangen', 'error') + return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id)) + + # Valideer JSON formaat + try: + parsed_json = json.loads(json_data) + except json.JSONDecodeError as e: + flash(f'Ongeldig JSON-formaat: {str(e)}', 'error') + return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id)) + + # Upload de bijgewerkte JSON naar MinIO + bucket_name, object_name, file_size = minio_client.upload_asset_file( + tenant_id, + asset.id, + asset.type, + asset.file_type, + json_data + ) + + # Update asset metadata + asset.file_size = file_size + asset.updated_at = dt.now(tz.utc) + asset.updated_by = current_user.id + asset.last_used_at = dt.now(tz.utc) + + db.session.commit() + + flash('Asset succesvol bijgewerkt', 'success') + return redirect(prefixed_url_for('interaction_bp.assets')) + + except Exception as e: + db.session.rollback() + current_app.logger.error(f"Error saving asset {asset_id}: {str(e)}") + flash(f'Fout bij het opslaan van het asset: {str(e)}', 'error') + return redirect(prefixed_url_for('interaction_bp.edit_asset', asset_id=asset_id)) \ No newline at end of file diff --git a/eveai_app/views/list_views/__init__.py b/eveai_app/views/list_views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eveai_app/views/list_views/assets_list_view.py b/eveai_app/views/list_views/assets_list_view.py new file mode 100644 index 0000000..ae8a075 --- /dev/null +++ b/eveai_app/views/list_views/assets_list_view.py @@ -0,0 +1,84 @@ +from datetime import datetime as dt, timezone as tz +from flask import request, render_template, current_app +from sqlalchemy import desc, asc +from common.models.interaction import EveAIAsset +from eveai_app.views.list_views.filtered_list_view import FilteredListView + + +class AssetsListView(FilteredListView): + allowed_filters = ['type', 'file_type'] + allowed_sorts = ['id', 'last_used_at'] + + def get_query(self): + return EveAIAsset.query + + def apply_filters(self, query): + filters = request.args.to_dict(flat=False) + + if 'type' in filters and filters['type']: + query = query.filter(EveAIAsset.type.in_(filters['type'])) + + if 'file_type' in filters and filters['file_type']: + query = query.filter(EveAIAsset.file_type.in_(filters['file_type'])) + + return query + + def apply_sorting(self, query): + sort_by = request.args.get('sort_by', 'id') + sort_order = request.args.get('sort_order', 'asc') + + if sort_by in self.allowed_sorts: + column = getattr(EveAIAsset, sort_by) + + if sort_order == 'asc': + query = query.order_by(asc(column)) + elif sort_order == 'desc': + query = query.order_by(desc(column)) + + return query + + def get(self): + query = self.get_query() + query = self.apply_filters(query) + query = self.apply_sorting(query) + pagination = self.paginate(query) + + def format_date(date): + if isinstance(date, dt): + return date.strftime('%Y-%m-%d %H:%M:%S') + elif isinstance(date, str): + return date + else: + return '' + + current_app.logger.debug(f"Assets retrieved: {pagination.items}") + rows = [ + [ + {'value': item.id, 'class': '', 'type': 'text'}, + {'value': item.name, 'class': '', 'type': 'text'}, + {'value': item.type, 'class': '', 'type': 'text'}, + {'value': item.type_version, 'class': '', 'type': 'text'}, + {'value': item.file_type or '', 'class': '', 'type': 'text'}, + {'value': format_date(item.last_used_at), 'class': '', 'type': 'text'} + ] for item in pagination.items + ] + + context = { + 'rows': rows, + 'pagination': pagination, + 'filters': request.args.to_dict(flat=False), + 'sort_by': request.args.get('sort_by', 'id'), + 'sort_order': request.args.get('sort_order', 'asc'), + 'filter_options': self.get_filter_options() + } + return render_template(self.template, **context) + + def get_filter_options(self): + # Haal unieke waarden op voor filters + types = [t[0] for t in EveAIAsset.query.with_entities(EveAIAsset.type).distinct().all() if t[0]] + file_types = [f[0] for f in EveAIAsset.query.with_entities(EveAIAsset.file_type).distinct().all() if f[0]] + + return { + 'type': [(t, t) for t in types], + 'file_type': [(f, f) for f in file_types] + } \ No newline at end of file diff --git a/eveai_app/views/document_list_view.py b/eveai_app/views/list_views/document_list_view.py similarity index 92% rename from eveai_app/views/document_list_view.py rename to eveai_app/views/list_views/document_list_view.py index c52513b..95f7910 100644 --- a/eveai_app/views/document_list_view.py +++ b/eveai_app/views/list_views/document_list_view.py @@ -1,9 +1,8 @@ from datetime import datetime as dt, timezone as tz from flask import request, render_template, session, current_app -from sqlalchemy import desc, asc, or_, and_, cast, Integer -from common.models.document import Document, Catalog -from common.utils.filtered_list_view import FilteredListView -from common.utils.view_assistants import prepare_table_for_macro +from sqlalchemy import desc, asc, or_, and_ +from common.models.document import Document +from eveai_app.views.list_views.filtered_list_view import FilteredListView class DocumentListView(FilteredListView): diff --git a/eveai_app/views/document_version_list_view.py b/eveai_app/views/list_views/document_version_list_view.py similarity index 96% rename from eveai_app/views/document_version_list_view.py rename to eveai_app/views/list_views/document_version_list_view.py index 4a5ee5d..9c860f7 100644 --- a/eveai_app/views/document_version_list_view.py +++ b/eveai_app/views/list_views/document_version_list_view.py @@ -1,9 +1,9 @@ from datetime import datetime -from flask import request, render_template, session +from flask import request, render_template from sqlalchemy import desc, asc from common.models.document import DocumentVersion, Document -from common.utils.filtered_list_view import FilteredListView +from eveai_app.views.list_views.filtered_list_view import FilteredListView from common.utils.view_assistants import prepare_table_for_macro diff --git a/common/utils/filtered_list_view.py b/eveai_app/views/list_views/filtered_list_view.py similarity index 100% rename from common/utils/filtered_list_view.py rename to eveai_app/views/list_views/filtered_list_view.py diff --git a/eveai_app/views/list_views/full_document_list_view.py b/eveai_app/views/list_views/full_document_list_view.py new file mode 100644 index 0000000..5c81107 --- /dev/null +++ b/eveai_app/views/list_views/full_document_list_view.py @@ -0,0 +1,179 @@ +from datetime import datetime as dt, timezone as tz +from flask import request, render_template, session, current_app +from sqlalchemy import desc, asc, or_, and_ +from sqlalchemy.orm import aliased + +from common.models.document import Document, DocumentVersion +from eveai_app.views.list_views.filtered_list_view import FilteredListView +from common.utils.view_assistants import prepare_table_for_macro + + +class FullDocumentListView(FilteredListView): + allowed_filters = ['validity', 'file_type', 'processing', 'processing_error'] + allowed_sorts = ['id', 'name', 'valid_from', 'valid_to', 'file_type', 'processing_started_at', + 'processing_finished_at', 'processing_error'] + + def __init__(self, model, template, per_page=10): + super().__init__(model, template, per_page) + self.version_alias = None + + def get_query(self): + catalog_id = session.get('catalog_id') + current_app.logger.debug(f"Catalog ID: {catalog_id}") + + # Fix: Selecteer alleen de id kolom in de subquery + latest_version_subquery = ( + DocumentVersion.query + .with_entities(DocumentVersion.id, DocumentVersion.doc_id, DocumentVersion.url, + DocumentVersion.bucket_name, DocumentVersion.object_name, + DocumentVersion.file_type, DocumentVersion.sub_file_type, + DocumentVersion.file_size, DocumentVersion.language, + DocumentVersion.user_context, DocumentVersion.system_context, + DocumentVersion.user_metadata, DocumentVersion.system_metadata, + DocumentVersion.catalog_properties, DocumentVersion.created_at, + DocumentVersion.created_by, DocumentVersion.updated_at, + DocumentVersion.updated_by, DocumentVersion.processing, + DocumentVersion.processing_started_at, DocumentVersion.processing_finished_at, + DocumentVersion.processing_error) + .filter(DocumentVersion.id == ( + DocumentVersion.query + .with_entities(DocumentVersion.id) # Selecteer alleen de id kolom + .filter(DocumentVersion.doc_id == Document.id) + .order_by(DocumentVersion.id.desc()) + .limit(1) + .scalar_subquery() + )) + .subquery() + ) + + self.version_alias = aliased(DocumentVersion, latest_version_subquery) + return Document.query.filter_by(catalog_id=catalog_id).outerjoin( + self.version_alias, Document.id == self.version_alias.doc_id + ) + + def apply_filters(self, query): + filters = request.args.to_dict(flat=False) + + # Document filters + if 'validity' in filters: + now = dt.now(tz.utc).date() + if 'valid' in filters['validity']: + query = query.filter( + and_( + or_(Document.valid_from.is_(None), Document.valid_from <= now), + or_(Document.valid_to.is_(None), Document.valid_to >= now) + ) + ) + + # DocumentVersion filters - use the same alias from get_query + if filters.get('file_type') and self.version_alias is not None: + query = query.filter(self.version_alias.file_type == filters['file_type'][0]) + + if filters.get('processing') and self.version_alias is not None: + query = query.filter(self.version_alias.processing == (filters['processing'][0] == 'true')) + + if filters.get('processing_error') and self.version_alias is not None: + if filters['processing_error'][0] == 'true': + query = query.filter(self.version_alias.processing_error.isnot(None)) + elif filters['processing_error'][0] == 'false': + query = query.filter(self.version_alias.processing_error.is_(None)) + + # Controleer of start_date een waarde heeft voordat we proberen te parsen + if filters.get('start_date') and self.version_alias is not None and filters['start_date'][0].strip(): + query = query.filter( + self.version_alias.processing_started_at >= dt.strptime(filters['start_date'][0], '%Y-%m-%d')) + + # Controleer of end_date een waarde heeft voordat we proberen te parsen + if filters.get('end_date') and self.version_alias is not None and filters['end_date'][0].strip(): + query = query.filter( + self.version_alias.processing_finished_at <= dt.strptime(filters['end_date'][0], '%Y-%m-%d')) + + return query + + def apply_sorting(self, query): + sort_by = request.args.get('sort_by', 'id') + sort_order = request.args.get('sort_order', 'asc') + + document_columns = ['id', 'name', 'valid_from', 'valid_to'] + version_columns = ['file_type', 'processing', 'processing_started_at', 'processing_finished_at', + 'processing_error'] + + if sort_by in self.allowed_sorts: + if sort_by in document_columns: + column = getattr(Document, sort_by) + elif sort_by in version_columns and self.version_alias is not None: + column = getattr(self.version_alias, sort_by) + else: + column = Document.id + + if sort_order == 'asc': + query = query.order_by(asc(column)) + elif sort_order == 'desc': + query = query.order_by(desc(column)) + + return query + + def get(self): + query = self.get_query() + query = self.apply_filters(query) + query = self.apply_sorting(query) + pagination = self.paginate(query) + + # Haal de laatste versies op voor elke document + items_with_versions = [] + for doc in pagination.items: + latest_version = DocumentVersion.query.filter_by(doc_id=doc.id).order_by(desc(DocumentVersion.id)).first() + items_with_versions.append((doc, latest_version)) + + def format_date(date): + if isinstance(date, dt): + return date.strftime('%Y-%m-%d') + elif isinstance(date, str): + return date + else: + return '' + + # Maak rijen voor de tabel met document en versie informatie + rows = [] + for doc, version in items_with_versions: + if version: + row = [ + {'value': doc.id, 'class': '', 'type': 'text'}, + {'value': doc.name, 'class': '', 'type': 'text'}, + {'value': format_date(doc.valid_from), 'class': '', 'type': 'text'}, + {'value': format_date(doc.valid_to), 'class': '', 'type': 'text'}, + {'value': version.id, 'class': '', 'type': 'text'}, + {'value': version.file_type, 'class': '', 'type': 'text'}, + {'value': 'Ja' if version.processing else 'Nee', 'class': '', 'type': 'text'}, + {'value': version.processing_error or '', 'class': '', 'type': 'text'} + ] + else: + row = [ + {'value': doc.id, 'class': '', 'type': 'text'}, + {'value': doc.name, 'class': '', 'type': 'text'}, + {'value': format_date(doc.valid_from), 'class': '', 'type': 'text'}, + {'value': format_date(doc.valid_to), 'class': '', 'type': 'text'}, + {'value': '', 'class': '', 'type': 'text'}, + {'value': '', 'class': '', 'type': 'text'}, + {'value': '', 'class': '', 'type': 'text'}, + {'value': '', 'class': '', 'type': 'text'} + ] + rows.append(row) + + context = { + 'rows': rows, + 'pagination': pagination, + 'filters': request.args.to_dict(flat=False), + 'sort_by': request.args.get('sort_by', 'id'), + 'sort_order': request.args.get('sort_order', 'asc'), + 'filter_options': self.get_filter_options() + } + return render_template(self.template, **context) + + def get_filter_options(self): + return { + 'validity': [('valid', 'Valid'), ('all', 'All')], + 'file_type': [('pdf', 'PDF'), ('docx', 'DOCX')], + 'processing': [('true', 'Processing'), ('false', 'Not Processing')], + 'processing_error': [('true', 'With Errors'), ('false', 'Without Errors')] + } diff --git a/eveai_chat_workers/specialists/traicie/TRAICIE_KO_INTERVIEW_DEFINITION_SPECIALIST/1_0.py b/eveai_chat_workers/specialists/traicie/TRAICIE_KO_INTERVIEW_DEFINITION_SPECIALIST/1_0.py index e2c1d50..1ba7a45 100644 --- a/eveai_chat_workers/specialists/traicie/TRAICIE_KO_INTERVIEW_DEFINITION_SPECIALIST/1_0.py +++ b/eveai_chat_workers/specialists/traicie/TRAICIE_KO_INTERVIEW_DEFINITION_SPECIALIST/1_0.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, Field from common.extensions import db, minio_client from common.models.interaction import Specialist, EveAIAsset +from common.utils.minio_utils import MIB_CONVERTOR from common.utils.eveai_exceptions import EveAISpecialistExecutionError from common.utils.model_logging_utils import set_logging_information from eveai_chat_workers.definitions.language_level.language_level_v1_0 import LANGUAGE_LEVEL @@ -209,7 +210,7 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor): # Stap 3: Storage metadata toevoegen asset.bucket_name = bucket_name asset.object_name = object_name - asset.file_size = file_size + asset.file_size = file_size / MIB_CONVERTOR asset.file_type = "json" # Stap 4: Token usage toevoegen