- Allowing for multiple types of Catalogs

- Introduction of retrievers
- Ensuring processing information is collected from Catalog iso Tenant
- Introduction of a generic Form class to enable dynamic fields based on a configuration
- Realisation of Retriever functionality to support dynamic fields

eveai_app/views/document_views.py

@@ -14,7 +14,7 @@ from urllib.parse import urlparse, unquote
 import io
 import json
 
-from common.models.document import Document, DocumentVersion, Catalog
+from common.models.document import Document, DocumentVersion, Catalog, Retriever
 from common.extensions import db, minio_client
 from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
     process_multiple_urls, get_documents_list, edit_document, \
@@ -22,7 +22,7 @@ from common.utils.document_utils import validate_file_type, create_document_stac
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException
 from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
-    CatalogForm
+    CatalogForm, RetrieverForm, EditRetrieverForm
 from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
@@ -30,6 +30,8 @@ from common.utils.view_assistants import form_validation_failed, prepare_table_f
 from .document_list_view import DocumentListView
 from .document_version_list_view import DocumentVersionListView
 
+from config.retriever_types import RETRIEVER_TYPES
+
 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
 
 
@@ -65,6 +67,7 @@ def catalog():
         tenant_id = session.get('tenant').get('id')
         new_catalog = Catalog()
         form.populate_obj(new_catalog)
+        new_catalog.parent_id = form.parent.data.get('id')
         # Handle Embedding Variables
         new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
         new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
@@ -103,7 +106,7 @@ def catalogs():
     the_catalogs = pagination.items
 
     # prepare table data
-    rows = prepare_table_for_macro(the_catalogs, [('id', ''), ('name', '')])
+    rows = prepare_table_for_macro(the_catalogs, [('id', ''), ('name', ''), ('type', '')])
 
     # Render the catalogs in a template
     return render_template('document/catalogs.html', rows=rows, pagination=pagination)
@@ -173,6 +176,121 @@ def edit_catalog(catalog_id):
     return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id)
 
 
+@document_bp.route('/retriever', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def retriever():
+    form = RetrieverForm()
+
+    if form.validate_on_submit():
+        tenant_id = session.get('tenant').get('id')
+        new_retriever = Retriever()
+        form.populate_obj(new_retriever)
+        new_retriever.catalog_id = form.catalog.data.id
+
+        set_logging_information(new_retriever, dt.now(tz.utc))
+
+        try:
+            db.session.add(new_retriever)
+            db.session.commit()
+            flash('Retriever successfully added!', 'success')
+            current_app.logger.info(f'Catalog {new_retriever.name} successfully added for tenant {tenant_id}!')
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash(f'Failed to add retriever. Error: {e}', 'danger')
+            current_app.logger.error(f'Failed to add retriever {new_retriever.name}'
+                                     f'for tenant {tenant_id}. Error: {str(e)}')
+
+        # Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
+        return redirect(prefixed_url_for('document_bp.retriever', retriever_id=new_retriever.id))
+
+    return render_template('document/retriever.html', form=form)
+
+
+@document_bp.route('/retriever/<int:retriever_id>', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def edit_retriever(retriever_id):
+    """Edit an existing retriever configuration."""
+    current_app.logger.debug(f"Editing Retriever {retriever_id}")
+
+    # Get the retriever or return 404
+    retriever = Retriever.query.get_or_404(retriever_id)
+
+    if retriever.catalog_id:
+        # If catalog_id is just an ID, fetch the Catalog object
+        retriever.catalog = Catalog.query.get(retriever.catalog_id)
+    else:
+        retriever.catalog = None
+
+    # Create form instance with the retriever
+    form = EditRetrieverForm(request.form, obj=retriever)
+
+    configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"]
+    current_app.logger.debug(f"Configuration {configuration_config}")
+    form.add_dynamic_fields("configuration", configuration_config, retriever.configuration)
+
+    if form.validate_on_submit():
+        # Update basic fields
+        form.populate_obj(retriever)
+        retriever.configuration = form.get_dynamic_data('configuration')
+
+        # Update catalog relationship
+        retriever.catalog_id = form.catalog.data.id if form.catalog.data else None
+
+        # Update logging information
+        update_logging_information(retriever, dt.now(tz.utc))
+
+        # Save changes to database
+        try:
+            db.session.add(retriever)
+            db.session.commit()
+            flash('Retriever updated successfully!', 'success')
+            current_app.logger.info(f'Retriever {retriever.id} updated successfully')
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash(f'Failed to update retriever. Error: {str(e)}', 'danger')
+            current_app.logger.error(f'Failed to update retriever {retriever_id}. Error: {str(e)}')
+            return render_template('document/edit_retriever.html', form=form, retriever_id=retriever_id)
+
+        return redirect(prefixed_url_for('document_bp.retrievers'))
+    else:
+        form_validation_failed(request, form)
+
+    current_app.logger.debug(f"Rendering Template for {retriever_id}")
+    return render_template('document/edit_retriever.html', form=form, retriever_id=retriever_id)
+
+
+@document_bp.route('/retrievers', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def retrievers():
+    page = request.args.get('page', 1, type=int)
+    per_page = request.args.get('per_page', 10, type=int)
+
+    query = Retriever.query.order_by(Retriever.id)
+
+    pagination = query.paginate(page=page, per_page=per_page)
+    the_retrievers = pagination.items
+
+    # prepare table data
+    rows = prepare_table_for_macro(the_retrievers,
+                                   [('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')])
+
+    # Render the catalogs in a template
+    return render_template('document/retrievers.html', rows=rows, pagination=pagination)
+
+
+@document_bp.route('/handle_retriever_selection', methods=['POST'])
+@roles_accepted('Super User', 'Tenant Admin')
+def handle_retriever_selection():
+    retriever_identification = request.form.get('selected_row')
+    retriever_id = ast.literal_eval(retriever_identification).get('value')
+    action = request.form['action']
+
+    if action == 'edit_retriever':
+        return redirect(prefixed_url_for('document_bp.edit_retriever', retriever_id=retriever_id))
+
+    return redirect(prefixed_url_for('document_bp.retrievers'))
+
+
 @document_bp.route('/add_document', methods=['GET', 'POST'])
 @roles_accepted('Super User', 'Tenant Admin')
 def add_document():