Add a DocumentVersion overview that can be sorted and can be filtered.

.idea/misc.xml

@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (eveAI)" project-jdk-type="Python SDK" />
+  <component name="Black">
+    <option name="sdkName" value="Python 3.12 (eveai_tbd)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (eveai_tbd)" project-jdk-type="Python SDK" />
 </project>

common/utils/filtered_list_view.py

@@ -0,0 +1,54 @@
+from flask import request, render_template, abort
+from sqlalchemy import desc, asc
+
+
+class FilteredListView:
+    def __init__(self, model, template, per_page=10):
+        self.model = model
+        self.template = template
+        self.per_page = per_page
+
+    def get_query(self):
+        return self.model.query
+
+    def apply_filters(self, query):
+        filters = request.args.get('filters', {})
+        for key, value in filters.items():
+            if hasattr(self.model, key):
+                column = getattr(self.model, key)
+                if value.startswith('like:'):
+                    query = query.filter(column.like(f"%{value[5:]}%"))
+                else:
+                    query = query.filter(column == value)
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by')
+        if sort_by and hasattr(self.model, sort_by):
+            sort_order = request.args.get('sort_order', 'asc')
+            column = getattr(self.model, sort_by)
+            if sort_order == 'desc':
+                query = query.order_by(desc(column))
+            else:
+                query = query.order_by(asc(column))
+        return query
+
+    def paginate(self, query):
+        page = request.args.get('page', 1, type=int)
+        return query.paginate(page=page, per_page=self.per_page, error_out=False)
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        context = {
+            'items': pagination.items,
+            'pagination': pagination,
+            'model': self.model.__name__,
+            'filters': request.args.get('filters', {}),
+            'sort_by': request.args.get('sort_by'),
+            'sort_order': request.args.get('sort_order', 'asc')
+        }
+        return render_template(self.template, **context)

common/utils/security_utils.py

@@ -5,9 +5,6 @@ from itsdangerous import URLSafeTimedSerializer
 from common.utils.nginx_utils import prefixed_url_for
 
 
-
-
-
 def confirm_token(token, expiration=3600):
     serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY'])
     try:
@@ -50,4 +47,3 @@ def send_reset_email(user):
     reset_url = prefixed_url_for('security_bp.reset_password', token=token, _external=True)
     html = render_template('email/reset_password.html', reset_url=reset_url)
     send_email(user.email, "Reset Your Password", html)
-

eveai_app/templates/document/document_versions_list_view.html

@@ -0,0 +1,83 @@
+{% extends 'base.html' %}
+{% from 'macros.html' import render_selectable_table, render_pagination, render_filter_field, render_date_filter_field, render_collapsible_section, render_selectable_sortable_table %}
+
+{% block title %}Documents{% endblock %}
+
+{% block content_title %}Document Versions{% endblock %}
+{% block content_description %}View Document Versions for Tenant{% endblock %}
+{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
+
+{% block content %}
+    <!-- Filter Form -->
+    {% set filter_form %}
+        <form method="GET" action="{{ url_for('document_bp.document_versions_list') }}">
+            {{ render_filter_field('file_type', 'File Type', filter_options['file_type'], filters.get('file_type')) }}
+            {{ render_filter_field('processing', 'Processing Status', filter_options['processing'], filters.get('processing')) }}
+            {{ render_filter_field('processing_error', 'Error Status', filter_options['processing_error'], filters.get('processing_error')) }}
+            {{ render_date_filter_field('start_date', 'Processing Start Date', filters.get('start_date')) }}
+            {{ render_date_filter_field('end_date', 'Processing End Date', filters.get('end_date')) }}
+
+            <button type="submit" class="btn btn-primary">Apply Filters</button>
+        </form>
+    {% endset %}
+
+    {{ render_collapsible_section('Filter', 'Filter Options', filter_form) }}
+
+    <!-- Document Versions Table -->
+    {{ render_selectable_sortable_table(
+        headers=["ID", "File Type", "Processing", "Processing Start", "Processing Finish", "Processing Error"],
+        rows=rows,
+        selectable=True,
+        id="documentVersionsTable",
+        sort_by=sort_by,
+        sort_order=sort_order
+    ) }}
+{% endblock %}
+
+{% block content_footer %}
+    {{ render_pagination(pagination, 'document_bp.document_versions_list') }}
+{% endblock %}
+
+{% block scripts %}
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    const table = document.getElementById('documentVersionsTable');
+    const headers = table.querySelectorAll('th.sortable');
+
+    headers.forEach(header => {
+        header.addEventListener('click', function() {
+            const sortBy = this.dataset.sort;
+            let sortOrder = 'asc';
+
+            if (this.querySelector('.fa-sort-up')) {
+                sortOrder = 'desc';
+            } else if (this.querySelector('.fa-sort-down')) {
+                sortOrder = 'none';
+            }
+
+            window.location.href = updateQueryStringParameter(window.location.href, 'sort_by', sortBy);
+            window.location.href = updateQueryStringParameter(window.location.href, 'sort_order', sortOrder);
+        });
+    });
+
+    function updateQueryStringParameter(uri, key, value) {
+        var re = new RegExp("([?&])" + key + "=.*?(&|$)", "i");
+        var separator = uri.indexOf('?') !== -1 ? "&" : "?";
+        if (uri.match(re)) {
+            return uri.replace(re, '$1' + key + "=" + value + '$2');
+        }
+        else {
+            return uri + separator + key + "=" + value;
+        }
+    }
+
+    table.addEventListener('change', function(event) {
+        if (event.target.type === 'radio') {
+            var selectedRow = event.target.closest('tr');
+            var documentVersionId = selectedRow.cells[1].textContent;
+            console.log('Selected Document Version ID:', documentVersionId);
+        }
+    });
+});
+</script>
+{% endblock %}

eveai_app/templates/macros.html

@@ -135,6 +135,48 @@
 </div>
 {% endmacro %}
 
+{% macro render_selectable_sortable_table(headers, rows, selectable, id, sort_by, sort_order) %}
+<div class="card">
+  <div class="table-responsive">
+    <table class="table align-items-center mb-0" id="{{ id }}">
+        <thead>
+            <tr>
+                {% if selectable %}
+                    <th>Select</th>
+                {% endif %}
+                {% for header in headers %}
+                    <th class="sortable" data-sort="{{ header|lower|replace(' ', '_') }}">
+                        {{ header }}
+                        {% if sort_by == header|lower|replace(' ', '_') %}
+                            {% if sort_order == 'asc' %}
+                                <i class="fas fa-sort-up"></i>
+                            {% elif sort_order == 'desc' %}
+                                <i class="fas fa-sort-down"></i>
+                            {% endif %}
+                        {% else %}
+                            <i class="fas fa-sort"></i>
+                        {% endif %}
+                    </th>
+                {% endfor %}
+            </tr>
+        </thead>
+        <tbody>
+            {% for row in rows %}
+                <tr>
+                    {% if selectable %}
+                        <td><input type="radio" name="selected_row" value="{{ row[0].value }}"></td>
+                    {% endif %}
+                    {% for cell in row %}
+                        <td>{{ cell.value }}</td>
+                    {% endfor %}
+                </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+  </div>
+</div>
+{% endmacro %}
+
 {% macro render_accordion(accordion_id, accordion_items, header_title, header_description) %}
 <div class="accordion-1">
   <div class="container">
@@ -253,3 +295,41 @@
 </nav>
 {% endmacro %}
 
+{% macro render_filter_field(field_name, label, options, current_value) %}
+<div class="form-group">
+    <label for="{{ field_name }}">{{ label }}</label>
+    <select class="form-control" id="{{ field_name }}" name="{{ field_name }}">
+        <option value="">All</option>
+        {% for value, text in options %}
+            <option value="{{ value }}" {% if value == current_value %}selected{% endif %}>{{ text }}</option>
+        {% endfor %}
+    </select>
+</div>
+{% endmacro %}
+
+{% macro render_date_filter_field(field_name, label, current_value) %}
+<div class="form-group">
+    <label for="{{ field_name }}">{{ label }}</label>
+    <input type="date" class="form-control" id="{{ field_name }}" name="{{ field_name }}" value="{{ current_value }}">
+</div>
+{% endmacro %}
+
+{% macro render_collapsible_section(id, title, content) %}
+<div class="accordion" id="accordion{{ id }}">
+    <div class="accordion-item mb-3">
+        <h5 class="accordion-header" id="heading{{ id }}">
+            <button class="accordion-button border-bottom font-weight-bold collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse{{ id }}" aria-expanded="false" aria-controls="collapse{{ id }}">
+                {{ title }}
+                <i class="collapse-close fa fa-plus text-xs pt-1 position-absolute end-0 me-3" aria-hidden="true"></i>
+                <i class="collapse-open fa fa-minus text-xs pt-1 position-absolute end-0 me-3" aria-hidden="true"></i>
+            </button>
+        </h5>
+        <div id="collapse{{ id }}" class="accordion-collapse collapse" aria-labelledby="heading{{ id }}" data-bs-parent="#accordion{{ id }}">
+            <div class="accordion-body text-sm opacity-8">
+                {{ content }}
+            </div>
+        </div>
+    </div>
+</div>
+{% endmacro %}
+

eveai_app/templates/navbar.html

@@ -86,6 +86,7 @@
                                     {'name': 'Add a list of URLs', 'url': '/document/add_urls', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Add Youtube Document' , 'url': '/document/add_youtube', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'All Documents', 'url': '/document/documents', 'roles': ['Super User', 'Tenant Admin']},
+                                    {'name': 'All Document Versions', 'url': '/document/document_versions_list', 'roles': ['Super User', 'Tenant Admin']},
                                     {'name': 'Library Operations', 'url': '/document/library_operations', 'roles': ['Super User', 'Tenant Admin']},
                                 ]) }}
                             {% endif %}

eveai_app/templates/scripts.html

@@ -12,3 +12,4 @@
         <script src="{{url_for('static', filename='assets/js/plugins/nouislider.min.js')}}"></script>
         <script src="{{url_for('static', filename='assets/js/plugins/anime.min.js')}}"></script>
         <script src="{{url_for('static', filename='assets/js/material-kit-pro.min.js')}}?v=3.0.4 type="text/javascript"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/js/bootstrap.bundle.min.js"></script>

eveai_app/views/document_version_list_view.py

@@ -0,0 +1,83 @@
+from datetime import datetime
+from flask import request, render_template, session
+from sqlalchemy import desc, asc
+
+from common.models.document import DocumentVersion, Document
+from common.utils.filtered_list_view import FilteredListView
+from common.utils.view_assistants import prepare_table_for_macro
+
+
+class DocumentVersionListView(FilteredListView):
+    allowed_filters = ['file_type', 'processing', 'processing_error']
+    allowed_sorts = ['id', 'processing_started_at', 'processing_finished_at', 'processing_error']
+
+    def get_query(self):
+        return DocumentVersion.query.join(Document).filter(Document.tenant_id == session.get('tenant', {}).get('id'))
+
+    def apply_filters(self, query):
+        filters = request.args.to_dict()
+
+        if filters.get('file_type'):
+            query = query.filter(DocumentVersion.file_type == filters['file_type'])
+
+        if filters.get('processing'):
+            query = query.filter(DocumentVersion.processing == (filters['processing'] == 'true'))
+
+        if filters.get('processing_error'):
+            if filters['processing_error'] == 'true':
+                query = query.filter(DocumentVersion.processing_error.isnot(None))
+            elif filters['processing_error'] == 'false':
+                query = query.filter(DocumentVersion.processing_error.is_(None))
+
+        if filters.get('start_date'):
+            query = query.filter(
+                DocumentVersion.processing_started_at >= datetime.strptime(filters['start_date'], '%Y-%m-%d'))
+
+        if filters.get('end_date'):
+            query = query.filter(
+                DocumentVersion.processing_finished_at <= datetime.strptime(filters['end_date'], '%Y-%m-%d'))
+
+        return query
+
+    def apply_sorting(self, query):
+        sort_by = request.args.get('sort_by', 'id')
+        sort_order = request.args.get('sort_order', 'asc')
+
+        if sort_by in self.allowed_sorts:
+            column = getattr(DocumentVersion, sort_by)
+            if sort_order == 'asc':
+                query = query.order_by(asc(column))
+            elif sort_order == 'desc':
+                query = query.order_by(desc(column))
+
+        return query
+
+    def get(self):
+        query = self.get_query()
+        query = self.apply_filters(query)
+        query = self.apply_sorting(query)
+        pagination = self.paginate(query)
+
+        rows = prepare_table_for_macro(
+            pagination.items,
+            [('id', ''), ('file_type', ''), ('processing', ''),
+             ('processing_started_at', ''), ('processing_finished_at', ''),
+             ('processing_error', '')]
+        )
+
+        context = {
+            'rows': rows,
+            'pagination': pagination,
+            'filters': request.args.to_dict(),
+            'sort_by': request.args.get('sort_by', 'id'),
+            'sort_order': request.args.get('sort_order', 'asc'),
+            'filter_options': self.get_filter_options()
+        }
+        return render_template(self.template, **context)
+
+    def get_filter_options(self):
+        return {
+            'file_type': [('pdf', 'PDF'), ('docx', 'DOCX')],
+            'processing': [('true', 'Processing'), ('false', 'Not Processing')],
+            'processing_error': [('true', 'With Errors'), ('false', 'Without Errors')]
+        }

eveai_app/views/document_views.py

@@ -24,6 +24,7 @@ from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
 from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro, form_to_dict
+from .document_version_list_view import DocumentVersionListView
 
 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
 
@@ -388,6 +389,15 @@ def handle_library_selection():
     return redirect(prefixed_url_for('document_bp.library_operations'))
 
 
+@document_bp.route('/document_versions_list', methods=['GET'])
+@roles_accepted('Super User', 'Tenant Admin')
+def document_versions_list():
+    current_app.logger.debug('Getting document versions list')
+    view = DocumentVersionListView(DocumentVersion, 'document/document_versions_list_view.html', per_page=20)
+    current_app.logger.debug('Got document versions list')
+    return view.get()
+
+
 def refresh_all_documents():
     for doc in Document.query.all():
         refresh_document(doc.id)