- Adding functionality for listing and editing assets

- Started adding functionality for creating a 'full_documents' list view.

eveai_app/views/list_views/assets_list_view.py

@@ -0,0 +1,84 @@
+from datetime import datetime as dt, timezone as tz
+from flask import request, render_template, current_app
+from sqlalchemy import desc, asc
+from common.models.interaction import EveAIAsset
+from eveai_app.views.list_views.filtered_list_view import FilteredListView
+
+
+class AssetsListView(FilteredListView):
+    allowed_filters = ['type', 'file_type']
+    allowed_sorts = ['id', 'last_used_at']
+
+    def get_query(self):
+        return EveAIAsset.query
+
+    def apply_filters(self, query):
+        filters = request.args.to_dict(flat=False)
+
+        if 'type' in filters and filters['type']:
+            query = query.filter(EveAIAsset.type.in_(filters['type']))
+
+        if 'file_type' in filters and filters['file_type']:
+            query = query.filter(EveAIAsset.file_type.in_(filters['file_type']))
+
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by', 'id')
+        sort_order = request.args.get('sort_order', 'asc')
+
+        if sort_by in self.allowed_sorts:
+            column = getattr(EveAIAsset, sort_by)
+
+            if sort_order == 'asc':
+                query = query.order_by(asc(column))
+            elif sort_order == 'desc':
+                query = query.order_by(desc(column))
+
+        return query
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        def format_date(date):
+            if isinstance(date, dt):
+                return date.strftime('%Y-%m-%d %H:%M:%S')
+            elif isinstance(date, str):
+                return date
+            else:
+                return ''
+
+        current_app.logger.debug(f"Assets retrieved: {pagination.items}")
+        rows = [
+            [
+                {'value': item.id, 'class': '', 'type': 'text'},
+                {'value': item.name, 'class': '', 'type': 'text'},
+                {'value': item.type, 'class': '', 'type': 'text'},
+                {'value': item.type_version, 'class': '', 'type': 'text'},
+                {'value': item.file_type or '', 'class': '', 'type': 'text'},
+                {'value': format_date(item.last_used_at), 'class': '', 'type': 'text'}
+            ] for item in pagination.items
+        ]
+
+        context = {
+            'rows': rows,
+            'pagination': pagination,
+            'filters': request.args.to_dict(flat=False),
+            'sort_by': request.args.get('sort_by', 'id'),
+            'sort_order': request.args.get('sort_order', 'asc'),
+            'filter_options': self.get_filter_options()
+        }
+        return render_template(self.template, **context)
+
+    def get_filter_options(self):
+        # Haal unieke waarden op voor filters
+        types = [t[0] for t in EveAIAsset.query.with_entities(EveAIAsset.type).distinct().all() if t[0]]
+        file_types = [f[0] for f in EveAIAsset.query.with_entities(EveAIAsset.file_type).distinct().all() if f[0]]
+
+        return {
+            'type': [(t, t) for t in types],
+            'file_type': [(f, f) for f in file_types]
+        }

eveai_app/views/list_views/document_list_view.py

@@ -0,0 +1,83 @@
+from datetime import datetime as dt, timezone as tz
+from flask import request, render_template, session, current_app
+from sqlalchemy import desc, asc, or_, and_
+from common.models.document import Document
+from eveai_app.views.list_views.filtered_list_view import FilteredListView
+
+
+class DocumentListView(FilteredListView):
+    allowed_filters = ['validity']
+    allowed_sorts = ['id', 'name', 'valid_from', 'valid_to']
+
+    def get_query(self):
+        catalog_id = session.get('catalog_id')
+        current_app.logger.debug(f"Catalog ID: {catalog_id}")
+        return Document.query.filter_by(catalog_id=catalog_id)
+
+    def apply_filters(self, query):
+        filters = request.args.to_dict(flat=False)
+
+        if 'validity' in filters:
+            now = dt.now(tz.utc).date()
+            if 'valid' in filters['validity']:
+                query = query.filter(
+                    and_(
+                        or_(Document.valid_from.is_(None), Document.valid_from <= now),
+                        or_(Document.valid_to.is_(None), Document.valid_to >= now)
+                    )
+                )
+
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by', 'id')
+        sort_order = request.args.get('sort_order', 'asc')
+
+        if sort_by in self.allowed_sorts:
+            column = getattr(Document, sort_by)
+
+            if sort_order == 'asc':
+                query = query.order_by(asc(column))
+            elif sort_order == 'desc':
+                query = query.order_by(desc(column))
+
+        return query
+
+    def get(self):
+        query = self.get_query()
+        # query = self.apply_filters(query)
+        # query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        def format_date(date):
+            if isinstance(date, dt):
+                return date.strftime('%Y-%m-%d')
+            elif isinstance(date, str):
+                return date
+            else:
+                return ''
+
+        current_app.logger.debug(f"Items retrieved: {pagination.items}")
+        rows = [
+            [
+                {'value': item.id, 'class': '', 'type': 'text'},
+                {'value': item.name, 'class': '', 'type': 'text'},
+                {'value': format_date(item.valid_from), 'class': '', 'type': 'text'},
+                {'value': format_date(item.valid_to), 'class': '', 'type': 'text'}
+            ] for item in pagination.items
+        ]
+
+        context = {
+            'rows': rows,
+            'pagination': pagination,
+            'filters': request.args.to_dict(flat=False),
+            'sort_by': request.args.get('sort_by', 'id'),
+            'sort_order': request.args.get('sort_order', 'asc'),
+            'filter_options': self.get_filter_options()
+        }
+        return render_template(self.template, **context)
+
+    def get_filter_options(self):
+        return {
+            'validity': [('valid', 'Valid'), ('all', 'All')]
+        }

eveai_app/views/list_views/document_version_list_view.py

@@ -0,0 +1,83 @@
+from datetime import datetime
+from flask import request, render_template
+from sqlalchemy import desc, asc
+
+from common.models.document import DocumentVersion, Document
+from eveai_app.views.list_views.filtered_list_view import FilteredListView
+from common.utils.view_assistants import prepare_table_for_macro
+
+
+class DocumentVersionListView(FilteredListView):
+    allowed_filters = ['file_type', 'processing', 'processing_error']
+    allowed_sorts = ['id', 'processing_started_at', 'processing_finished_at', 'processing_error']
+
+    def get_query(self):
+        return DocumentVersion.query.join(Document)
+
+    def apply_filters(self, query):
+        filters = request.args.to_dict()
+
+        if filters.get('file_type'):
+            query = query.filter(DocumentVersion.file_type == filters['file_type'])
+
+        if filters.get('processing'):
+            query = query.filter(DocumentVersion.processing == (filters['processing'] == 'true'))
+
+        if filters.get('processing_error'):
+            if filters['processing_error'] == 'true':
+                query = query.filter(DocumentVersion.processing_error.isnot(None))
+            elif filters['processing_error'] == 'false':
+                query = query.filter(DocumentVersion.processing_error.is_(None))
+
+        if filters.get('start_date'):
+            query = query.filter(
+                DocumentVersion.processing_started_at >= datetime.strptime(filters['start_date'], '%Y-%m-%d'))
+
+        if filters.get('end_date'):
+            query = query.filter(
+                DocumentVersion.processing_finished_at <= datetime.strptime(filters['end_date'], '%Y-%m-%d'))
+
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by', 'id')
+        sort_order = request.args.get('sort_order', 'asc')
+
+        if sort_by in self.allowed_sorts:
+            column = getattr(DocumentVersion, sort_by)
+            if sort_order == 'asc':
+                query = query.order_by(asc(column))
+            elif sort_order == 'desc':
+                query = query.order_by(desc(column))
+
+        return query
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        rows = prepare_table_for_macro(
+            pagination.items,
+            [('id', ''), ('file_type', ''), ('processing', ''),
+             ('processing_started_at', ''), ('processing_finished_at', ''),
+             ('processing_error', '')]
+        )
+
+        context = {
+            'rows': rows,
+            'pagination': pagination,
+            'filters': request.args.to_dict(),
+            'sort_by': request.args.get('sort_by', 'id'),
+            'sort_order': request.args.get('sort_order', 'asc'),
+            'filter_options': self.get_filter_options()
+        }
+        return render_template(self.template, **context)
+
+    def get_filter_options(self):
+        return {
+            'file_type': [('pdf', 'PDF'), ('docx', 'DOCX')],
+            'processing': [('true', 'Processing'), ('false', 'Not Processing')],
+            'processing_error': [('true', 'With Errors'), ('false', 'Without Errors')]
+        }

eveai_app/views/list_views/filtered_list_view.py

@@ -0,0 +1,54 @@
+from flask import request, render_template, abort
+from sqlalchemy import desc, asc
+
+
+class FilteredListView:
+    def __init__(self, model, template, per_page=10):
+        self.model = model
+        self.template = template
+        self.per_page = per_page
+
+    def get_query(self):
+        return self.model.query
+
+    def apply_filters(self, query):
+        filters = request.args.get('filters', {})
+        for key, value in filters.items():
+            if hasattr(self.model, key):
+                column = getattr(self.model, key)
+                if value.startswith('like:'):
+                    query = query.filter(column.like(f"%{value[5:]}%"))
+                else:
+                    query = query.filter(column == value)
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by')
+        if sort_by and hasattr(self.model, sort_by):
+            sort_order = request.args.get('sort_order', 'asc')
+            column = getattr(self.model, sort_by)
+            if sort_order == 'desc':
+                query = query.order_by(desc(column))
+            else:
+                query = query.order_by(asc(column))
+        return query
+
+    def paginate(self, query):
+        page = request.args.get('page', 1, type=int)
+        return query.paginate(page=page, per_page=self.per_page, error_out=False)
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        context = {
+            'items': pagination.items,
+            'pagination': pagination,
+            'model': self.model.__name__,
+            'filters': request.args.get('filters', {}),
+            'sort_by': request.args.get('sort_by'),
+            'sort_order': request.args.get('sort_order', 'asc')
+        }
+        return render_template(self.template, **context)

eveai_app/views/list_views/full_document_list_view.py

@@ -0,0 +1,179 @@
+from datetime import datetime as dt, timezone as tz
+from flask import request, render_template, session, current_app
+from sqlalchemy import desc, asc, or_, and_
+from sqlalchemy.orm import aliased
+
+from common.models.document import Document, DocumentVersion
+from eveai_app.views.list_views.filtered_list_view import FilteredListView
+from common.utils.view_assistants import prepare_table_for_macro
+
+
+class FullDocumentListView(FilteredListView):
+    allowed_filters = ['validity', 'file_type', 'processing', 'processing_error']
+    allowed_sorts = ['id', 'name', 'valid_from', 'valid_to', 'file_type', 'processing_started_at',
+                     'processing_finished_at', 'processing_error']
+
+    def __init__(self, model, template, per_page=10):
+        super().__init__(model, template, per_page)
+        self.version_alias = None
+
+    def get_query(self):
+        catalog_id = session.get('catalog_id')
+        current_app.logger.debug(f"Catalog ID: {catalog_id}")
+
+        # Fix: Selecteer alleen de id kolom in de subquery
+        latest_version_subquery = (
+            DocumentVersion.query
+            .with_entities(DocumentVersion.id, DocumentVersion.doc_id, DocumentVersion.url,
+                           DocumentVersion.bucket_name, DocumentVersion.object_name,
+                           DocumentVersion.file_type, DocumentVersion.sub_file_type,
+                           DocumentVersion.file_size, DocumentVersion.language,
+                           DocumentVersion.user_context, DocumentVersion.system_context,
+                           DocumentVersion.user_metadata, DocumentVersion.system_metadata,
+                           DocumentVersion.catalog_properties, DocumentVersion.created_at,
+                           DocumentVersion.created_by, DocumentVersion.updated_at,
+                           DocumentVersion.updated_by, DocumentVersion.processing,
+                           DocumentVersion.processing_started_at, DocumentVersion.processing_finished_at,
+                           DocumentVersion.processing_error)
+            .filter(DocumentVersion.id == (
+                DocumentVersion.query
+                .with_entities(DocumentVersion.id)  # Selecteer alleen de id kolom
+                .filter(DocumentVersion.doc_id == Document.id)
+                .order_by(DocumentVersion.id.desc())
+                .limit(1)
+                .scalar_subquery()
+            ))
+            .subquery()
+        )
+
+        self.version_alias = aliased(DocumentVersion, latest_version_subquery)
+        return Document.query.filter_by(catalog_id=catalog_id).outerjoin(
+            self.version_alias, Document.id == self.version_alias.doc_id
+        )
+
+    def apply_filters(self, query):
+        filters = request.args.to_dict(flat=False)
+
+        # Document filters
+        if 'validity' in filters:
+            now = dt.now(tz.utc).date()
+            if 'valid' in filters['validity']:
+                query = query.filter(
+                    and_(
+                        or_(Document.valid_from.is_(None), Document.valid_from <= now),
+                        or_(Document.valid_to.is_(None), Document.valid_to >= now)
+                    )
+                )
+
+        # DocumentVersion filters - use the same alias from get_query
+        if filters.get('file_type') and self.version_alias is not None:
+            query = query.filter(self.version_alias.file_type == filters['file_type'][0])
+
+        if filters.get('processing') and self.version_alias is not None:
+            query = query.filter(self.version_alias.processing == (filters['processing'][0] == 'true'))
+
+        if filters.get('processing_error') and self.version_alias is not None:
+            if filters['processing_error'][0] == 'true':
+                query = query.filter(self.version_alias.processing_error.isnot(None))
+            elif filters['processing_error'][0] == 'false':
+                query = query.filter(self.version_alias.processing_error.is_(None))
+
+        # Controleer of start_date een waarde heeft voordat we proberen te parsen
+        if filters.get('start_date') and self.version_alias is not None and filters['start_date'][0].strip():
+            query = query.filter(
+                self.version_alias.processing_started_at >= dt.strptime(filters['start_date'][0], '%Y-%m-%d'))
+
+        # Controleer of end_date een waarde heeft voordat we proberen te parsen
+        if filters.get('end_date') and self.version_alias is not None and filters['end_date'][0].strip():
+            query = query.filter(
+                self.version_alias.processing_finished_at <= dt.strptime(filters['end_date'][0], '%Y-%m-%d'))
+
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by', 'id')
+        sort_order = request.args.get('sort_order', 'asc')
+
+        document_columns = ['id', 'name', 'valid_from', 'valid_to']
+        version_columns = ['file_type', 'processing', 'processing_started_at', 'processing_finished_at',
+                           'processing_error']
+
+        if sort_by in self.allowed_sorts:
+            if sort_by in document_columns:
+                column = getattr(Document, sort_by)
+            elif sort_by in version_columns and self.version_alias is not None:
+                column = getattr(self.version_alias, sort_by)
+            else:
+                column = Document.id
+
+            if sort_order == 'asc':
+                query = query.order_by(asc(column))
+            elif sort_order == 'desc':
+                query = query.order_by(desc(column))
+
+        return query
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        # Haal de laatste versies op voor elke document
+        items_with_versions = []
+        for doc in pagination.items:
+            latest_version = DocumentVersion.query.filter_by(doc_id=doc.id).order_by(desc(DocumentVersion.id)).first()
+            items_with_versions.append((doc, latest_version))
+
+        def format_date(date):
+            if isinstance(date, dt):
+                return date.strftime('%Y-%m-%d')
+            elif isinstance(date, str):
+                return date
+            else:
+                return ''
+
+        # Maak rijen voor de tabel met document en versie informatie
+        rows = []
+        for doc, version in items_with_versions:
+            if version:
+                row = [
+                    {'value': doc.id, 'class': '', 'type': 'text'},
+                    {'value': doc.name, 'class': '', 'type': 'text'},
+                    {'value': format_date(doc.valid_from), 'class': '', 'type': 'text'},
+                    {'value': format_date(doc.valid_to), 'class': '', 'type': 'text'},
+                    {'value': version.id, 'class': '', 'type': 'text'},
+                    {'value': version.file_type, 'class': '', 'type': 'text'},
+                    {'value': 'Ja' if version.processing else 'Nee', 'class': '', 'type': 'text'},
+                    {'value': version.processing_error or '', 'class': '', 'type': 'text'}
+                ]
+            else:
+                row = [
+                    {'value': doc.id, 'class': '', 'type': 'text'},
+                    {'value': doc.name, 'class': '', 'type': 'text'},
+                    {'value': format_date(doc.valid_from), 'class': '', 'type': 'text'},
+                    {'value': format_date(doc.valid_to), 'class': '', 'type': 'text'},
+                    {'value': '', 'class': '', 'type': 'text'},
+                    {'value': '', 'class': '', 'type': 'text'},
+                    {'value': '', 'class': '', 'type': 'text'},
+                    {'value': '', 'class': '', 'type': 'text'}
+                ]
+            rows.append(row)
+
+        context = {
+            'rows': rows,
+            'pagination': pagination,
+            'filters': request.args.to_dict(flat=False),
+            'sort_by': request.args.get('sort_by', 'id'),
+            'sort_order': request.args.get('sort_order', 'asc'),
+            'filter_options': self.get_filter_options()
+        }
+        return render_template(self.template, **context)
+
+    def get_filter_options(self):
+        return {
+            'validity': [('valid', 'Valid'), ('all', 'All')],
+            'file_type': [('pdf', 'PDF'), ('docx', 'DOCX')],
+            'processing': [('true', 'Processing'), ('false', 'Not Processing')],
+            'processing_error': [('true', 'With Errors'), ('false', 'Without Errors')]
+        }