- Modernized authentication with the introduction of TenantProject

- Created a base mail template
- Adapt and improve document API to usage of catalogs and processors
- Adapt eveai_sync to new authentication mechanism and usage of catalogs and processors

eveai_app/views/document_views.py

@@ -1,7 +1,6 @@
 import ast
 from datetime import datetime as dt, timezone as tz
 
-from babel.messages.setuptools_frontend import update_catalog
 from flask import request, redirect, flash, render_template, Blueprint, session, current_app
 from flask_security import roles_accepted, current_user
 from sqlalchemy import desc
@@ -10,35 +9,33 @@ from werkzeug.utils import secure_filename
 from sqlalchemy.exc import SQLAlchemyError
 import requests
 from requests.exceptions import SSLError
-from urllib.parse import urlparse, unquote
-import io
 import json
 
 from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor
-from common.extensions import db, minio_client
+from common.extensions import db
 from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
-    process_multiple_urls, get_documents_list, edit_document, \
+    edit_document, \
     edit_document_version, refresh_document
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException
-from config.processor_types import PROCESSOR_TYPES
-from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
+from config.type_defs.processor_types import PROCESSOR_TYPES
+from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, \
     CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
 from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
-from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro, form_to_dict
+from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
 from .document_list_view import DocumentListView
 from .document_version_list_view import DocumentVersionListView
-from config.catalog_types import CATALOG_TYPES
-from config.retriever_types import RETRIEVER_TYPES
+from config.type_defs.catalog_types import CATALOG_TYPES
+from config.type_defs.retriever_types import RETRIEVER_TYPES
 
 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
 
 
 @document_bp.before_request
 def log_before_request():
-    pass
+    current_app.logger.debug(f'Before request: {request.path} =====================================')
 
 
 @document_bp.after_request
@@ -214,7 +211,7 @@ def edit_processor(processor_id):
             db.session.add(processor)
             db.session.commit()
             flash('Retriever updated successfully!', 'success')
-            current_app.logger.info(f'Retriever {processor.id} updated successfully')
+            current_app.logger.info(f'Processor {processor.id} updated successfully')
         except SQLAlchemyError as e:
             db.session.rollback()
             flash(f'Failed to update processor. Error: {str(e)}', 'danger')
@@ -376,7 +373,7 @@ def add_document():
     form = AddDocumentForm(request.form)
     catalog_id = session.get('catalog_id', None)
     if catalog_id is None:
-        flash('You need to set a Session Catalog before adding Documents or URLs')
+        flash('You need to set a Session Catalog before adding Documents or URLs', 'warning')
         return redirect(prefixed_url_for('document_bp.catalogs'))
 
     catalog = Catalog.query.get_or_404(catalog_id)
@@ -434,7 +431,7 @@ def add_url():
     form = AddURLForm(request.form)
     catalog_id = session.get('catalog_id', None)
     if catalog_id is None:
-        flash('You need to set a Session Catalog before adding Documents or URLs')
+        flash('You need to set a Session Catalog before adding Documents or URLs', 'warning')
         return redirect(prefixed_url_for('document_bp.catalogs'))
 
     catalog = Catalog.query.get_or_404(catalog_id)
@@ -547,6 +544,7 @@ def edit_document_view(document_id):
 
     if form.validate_on_submit():
         updated_doc, error = edit_document(
+            session.get('tenant').get('id', 0),
             document_id,
             form.name.data,
             form.valid_from.data,
@@ -569,10 +567,8 @@ def edit_document_version_view(document_version_id):
     doc_vers = DocumentVersion.query.get_or_404(document_version_id)
     form = EditDocumentVersionForm(request.form, obj=doc_vers)
 
-    catalog_id = session.get('catalog_id', None)
-    if catalog_id is None:
-        flash('You need to set a Session Catalog before adding Documents or URLs')
-        return redirect(prefixed_url_for('document_bp.catalogs'))
+    doc_vers = DocumentVersion.query.get_or_404(document_version_id)
+    catalog_id = doc_vers.document.catalog_id
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
@@ -587,6 +583,7 @@ def edit_document_version_view(document_version_id):
             catalog_properties[config] = form.get_dynamic_data(config)
 
         updated_version, error = edit_document_version(
+            session.get('tenant').get('id', 0),
             document_version_id,
             form.user_context.data,
             catalog_properties,