pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Introduction of dynamic Retrievers & Specialists

Browse Source 
- Introduction of dynamic Processors
- Introduction of caching system
- Introduction of a better template manager
- Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists
- Start adaptation of chat client
This commit is contained in:
Josako
2024-11-15 10:00:53 +01:00
parent 55a8a95f79
commit 1807435339
101 changed files with 4181 additions and 1764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
eveai_app/views/document_views.py
View File

@@ -14,15 +14,16 @@ from urllib.parse import urlparse, unquote
import io
import json
from common.models.document import Document, DocumentVersion, Catalog, Retriever
from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor
from common.extensions import db, minio_client
from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
process_multiple_urls, get_documents_list, edit_document, \
edit_document_version, refresh_document
from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
EveAIDoubleURLException
from config.processor_types import PROCESSOR_TYPES
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
@@ -37,13 +38,11 @@ document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@document_bp.before_request
def log_before_request():
current_app.logger.debug(f"Before request (document_bp): {request.method} {request.url}")
pass
@document_bp.after_request
def log_after_request(response):
current_app.logger.debug(
f"After request (document_bp): {request.method} {request.url} - Status: {response.status}")
return response
@@ -53,8 +52,6 @@ def before_request():
mw_before_request()
except Exception as e:
current_app.logger.error(f'Error switching schema in Document Blueprint: {e}')
for role in current_user.roles:
current_app.logger.debug(f'User {current_user.email} has role {role.name}')
raise
@@ -67,16 +64,6 @@ def catalog():
tenant_id = session.get('tenant').get('id')
new_catalog = Catalog()
form.populate_obj(new_catalog)
# Handle Embedding Variables
new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
if form.html_end_tags.data else []
new_catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
if form.html_included_elements.data else []
new_catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
if form.html_excluded_elements.data else []
new_catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
set_logging_information(new_catalog, dt.now(tz.utc))
try:
@@ -84,6 +71,8 @@ def catalog():
db.session.commit()
flash('Catalog successfully added!', 'success')
current_app.logger.info(f'Catalog {new_catalog.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of catalog - add configuration of the catalog (dependent on type)
return redirect(prefixed_url_for('document_bp.catalog', catalog_id=new_catalog.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add catalog. Error: {e}', 'danger')
@@ -140,27 +129,8 @@ def edit_catalog(catalog_id):
configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
# Convert arrays to comma-separated strings for display
if request.method == 'GET':
form.html_tags.data = ', '.join(catalog.html_tags or '')
form.html_end_tags.data = ', '.join(catalog.html_end_tags or '')
form.html_included_elements.data = ', '.join(catalog.html_included_elements or '')
form.html_excluded_elements.data = ', '.join(catalog.html_excluded_elements or '')
form.html_excluded_classes.data = ', '.join(catalog.html_excluded_classes or '')
if request.method == 'POST' and form.validate_on_submit():
form.populate_obj(catalog)
# Handle Embedding Variables
catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
if form.html_end_tags.data else []
catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
if form.html_included_elements.data else []
catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
if form.html_excluded_elements.data else []
catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
catalog.configuration = form.get_dynamic_data('configuration')
update_logging_information(catalog, dt.now(tz.utc))
try:
@@ -180,6 +150,116 @@ def edit_catalog(catalog_id):
return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id)
@document_bp.route('/processor', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def processor():
form = ProcessorForm()
if form.validate_on_submit():
tenant_id = session.get('tenant').get('id')
new_processor = Processor()
form.populate_obj(new_processor)
new_processor.catalog_id = form.catalog.data.id
set_logging_information(new_processor, dt.now(tz.utc))
try:
db.session.add(new_processor)
db.session.commit()
flash('Processor successfully added!', 'success')
current_app.logger.info(f'Processor {new_processor.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=new_processor.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add processor. Error: {e}', 'danger')
current_app.logger.error(f'Failed to add retriever {new_processor.name}'
f'for tenant {tenant_id}. Error: {str(e)}')
return render_template('document/processor.html', form=form)
@document_bp.route('/processor/<int:processor_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def edit_processor(processor_id):
"""Edit an existing processorr configuration."""
# Get the processor or return 404
processor = Processor.query.get_or_404(processor_id)
if processor.catalog_id:
# If catalog_id is just an ID, fetch the Catalog object
processor.catalog = Catalog.query.get(processor.catalog_id)
else:
processor.catalog = None
# Create form instance with the processor
form = EditProcessorForm(request.form, obj=processor)
configuration_config = PROCESSOR_TYPES[processor.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, processor.configuration)
if form.validate_on_submit():
# Update basic fields
form.populate_obj(processor)
processor.configuration = form.get_dynamic_data('configuration')
# Update catalog relationship
processor.catalog_id = form.catalog.data.id if form.catalog.data else None
# Update logging information
update_logging_information(processor, dt.now(tz.utc))
# Save changes to database
try:
db.session.add(processor)
db.session.commit()
flash('Retriever updated successfully!', 'success')
current_app.logger.info(f'Retriever {processor.id} updated successfully')
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to update processor. Error: {str(e)}', 'danger')
current_app.logger.error(f'Failed to update processor {processor_id}. Error: {str(e)}')
return render_template('document/edit_processor.html', form=form, processor_id=processor_id)
return redirect(prefixed_url_for('document_bp.processors'))
else:
form_validation_failed(request, form)
return render_template('document/edit_processor.html', form=form, processor_id=processor_id)
@document_bp.route('/processors', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def processors():
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = Processor.query.order_by(Processor.id)
pagination = query.paginate(page=page, per_page=per_page)
the_processors = pagination.items
# prepare table data
rows = prepare_table_for_macro(the_processors,
[('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')])
# Render the catalogs in a template
return render_template('document/processors.html', rows=rows, pagination=pagination)
@document_bp.route('/handle_processor_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def handle_processor_selection():
processor_identification = request.form.get('selected_row')
processor_id = ast.literal_eval(processor_identification).get('value')
action = request.form['action']
if action == 'edit_processor':
return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=processor_id))
return redirect(prefixed_url_for('document_bp.processors'))
@document_bp.route('/retriever', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def retriever():
@@ -198,15 +278,14 @@ def retriever():
db.session.commit()
flash('Retriever successfully added!', 'success')
current_app.logger.info(f'Catalog {new_retriever.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.edit_retriever', retriever_id=new_retriever.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add retriever. Error: {e}', 'danger')
current_app.logger.error(f'Failed to add retriever {new_retriever.name}'
f'for tenant {tenant_id}. Error: {str(e)}')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.retriever', retriever_id=new_retriever.id))
return render_template('document/retriever.html', form=form)
@@ -311,6 +390,7 @@ def add_document():
current_app.logger.info(f'Adding Document for {catalog_id}')
tenant_id = session['tenant']['id']
file = form.file.data
sub_file_type = form.sub_file_type.data
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
@@ -324,14 +404,13 @@ def add_document():
api_input = {
'catalog_id': catalog_id,
'name': form.name.data,
'sub_file_type': form.sub_file_type.data,
'language': form.language.data,
'user_context': form.user_context.data,
'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
'catalog_properties': catalog_properties,
}
current_app.logger.debug(f'Creating document stack with input {api_input}')
new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
task_id = start_embedding_task(tenant_id, new_doc_vers.id)
@@ -341,6 +420,7 @@ def add_document():
except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e:
flash(str(e), 'error')
current_app.logger.error(f"Error adding document: {str(e)}")
except Exception as e:
current_app.logger.error(f'Error adding document: {str(e)}')
flash('An error occurred while adding the document.', 'error')
@@ -378,6 +458,7 @@ def add_url():
api_input = {
'catalog_id': catalog_id,
'name': form.name.data or filename,
'sub_file_type': form.sub_file_type.data,
'url': url,
'language': form.language.data,
'user_context': form.user_context.data,
@@ -562,8 +643,6 @@ def handle_document_version_selection():
action = request.form['action']
current_app.logger.debug(f'Triggered Document Version Action: {action}')
match action:
case 'edit_document_version':
return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=doc_vers_id))
@@ -598,9 +677,7 @@ def handle_library_selection():
@document_bp.route('/document_versions_list', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin')
def document_versions_list():
current_app.logger.debug('Getting document versions list')
view = DocumentVersionListView(DocumentVersion, 'document/document_versions_list_view.html', per_page=20)
current_app.logger.debug('Got document versions list')
return view.get()
@@ -653,8 +730,9 @@ def update_logging_information(obj, timestamp):
def log_session_state(session, msg=""):
current_app.logger.debug(f"{msg} - Session dirty: {session.dirty}")
current_app.logger.debug(f"{msg} - Session new: {session.new}")
pass
# current_app.logger.info(f"{msg} - Session dirty: {session.dirty}")
# current_app.logger.info(f"{msg} - Session new: {session.new}")
def fetch_html(url):