pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Introduction of dynamic Retrievers & Specialists

Browse Source 
- Introduction of dynamic Processors
- Introduction of caching system
- Introduction of a better template manager
- Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists
- Start adaptation of chat client
This commit is contained in:
Josako
2024-11-15 10:00:53 +01:00
parent 55a8a95f79
commit 1807435339
101 changed files with 4181 additions and 1764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
eveai_app/__init__.py
View File

@@ -7,7 +7,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix
import logging.config
from common.extensions import (db, migrate, bootstrap, security, mail, login_manager, cors, csrf, session,
minio_client, simple_encryption, metrics)
minio_client, simple_encryption, metrics, cache_manager)
from common.models.user import User, Role, Tenant, TenantDomain
import common.models.interaction
import common.models.entitlements
@@ -119,6 +119,7 @@ def register_extensions(app):
simple_encryption.init_app(app)
session.init_app(app)
minio_client.init_app(app)
cache_manager.init_app(app)
metrics.init_app(app)

33
eveai_app/templates/document/edit_processor.html Normal file
View File

@@ -0,0 +1,33 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% block title %}Edit Processor{% endblock %}
{% block content_title %}Edit Processor{% endblock %}
{% block content_description %}Edit a Processor (for a Catalog){% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = ['type'] %}
{% set exclude_fields = [] %}
<!-- Render Static Fields -->
{% for field in form.get_static_fields() %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<!-- Render Dynamic Fields -->
{% for collection_name, fields in form.get_dynamic_fields().items() %}
{% if fields|length > 0 %}
<h4 class="mt-4">{{ collection_name }}</h4>
{% endif %}
{% for field in fields %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
{% endfor %}
<button type="submit" class="btn btn-primary">Save Processor</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

23
eveai_app/templates/document/processor.html Normal file
View File

@@ -0,0 +1,23 @@
{% extends 'base.html' %}
{% from "macros.html" import render_field %}
{% block title %}Processor Registration{% endblock %}
{% block content_title %}Register Processor{% endblock %}
{% block content_description %}Define a new processor (for a catalog){% endblock %}
{% block content %}
<form method="post">
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
{% for field in form %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
<button type="submit" class="btn btn-primary">Register Processor</button>
</form>
{% endblock %}
{% block content_footer %}
{% endblock %}

23
eveai_app/templates/document/processors.html Normal file
View File

@@ -0,0 +1,23 @@
{% extends 'base.html' %}
{% from 'macros.html' import render_selectable_table, render_pagination %}
{% block title %}Processors{% endblock %}
{% block content_title %}Processors{% endblock %}
{% block content_description %}View Processors for Tenant{% endblock %}
{% block content_class %}<div class="col-xl-12 col-lg-5 col-md-7 mx-auto"></div>{% endblock %}
{% block content %}
<div class="container">
<form method="POST" action="{{ url_for('document_bp.handle_processor_selection') }}">
{{ render_selectable_table(headers=["Processor ID", "Name", "Type", "Catalog ID"], rows=rows, selectable=True, id="retrieversTable") }}
<div class="form-group mt-3">
<button type="submit" name="action" value="edit_processor" class="btn btn-primary">Edit Processor</button>
</div>
</form>
</div>
{% endblock %}
{% block content_footer %}
{{ render_pagination(pagination, 'document_bp.processors') }}
{% endblock %}

2
eveai_app/templates/interaction/edit_specialist.html
View File

@@ -24,7 +24,7 @@
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
{% endfor %}
<button type="submit" class="btn btn-primary">Save Retriever</button>
<button type="submit" class="btn btn-primary">Save Specialist</button>
</form>
{% endblock %}

2
eveai_app/templates/navbar.html
View File

@@ -83,6 +83,8 @@
{{ dropdown('Document Mgmt', 'note_stack', [
{'name': 'Add Catalog', 'url': '/document/catalog', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'All Catalogs', 'url': '/document/catalogs', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Add Processor', 'url': '/document/processor', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'All Processors', 'url': '/document/processors', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Add Retriever', 'url': '/document/retriever', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'All Retrievers', 'url': '/document/retrievers', 'roles': ['Super User', 'Tenant Admin']},
{'name': 'Add Document', 'url': '/document/add_document', 'roles': ['Super User', 'Tenant Admin']},

3
eveai_app/views/basic_views.py
View File

@@ -9,12 +9,11 @@ basic_bp = Blueprint('basic_bp', __name__)
@basic_bp.before_request
def log_before_request():
current_app.logger.debug(f"Before request (basic_bp): {request.method} {request.url}")
pass
@basic_bp.after_request
def log_after_request(response):
current_app.logger.debug(f"After request (basic_bp): {request.method} {request.url} - Status: {response.status}")
return response

117
eveai_app/views/document_forms.py
View File

@@ -11,6 +11,7 @@ from wtforms_sqlalchemy.fields import QuerySelectField
from common.models.document import Catalog
from config.catalog_types import CATALOG_TYPES
from config.processor_types import PROCESSOR_TYPES
from config.retriever_types import RETRIEVER_TYPES
from .dynamic_form_base import DynamicFormBase
@@ -38,28 +39,14 @@ class CatalogForm(FlaskForm):
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = SelectField('Catalog Type', validators=[DataRequired()])
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
# HTML Embedding Variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
default='p, li')
html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
default='header, footer, nav, script')
html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
# Chat Variables
chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)])
chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)])
# Tuning variables
embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -74,28 +61,75 @@ class EditCatalogForm(DynamicFormBase):
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = StringField('Catalog Type', validators=[DataRequired()], render_kw={'readonly': True})
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],)
# HTML Embedding Variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
default='p, li')
html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
default='header, footer, nav, script')
html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
# Chat Variables
chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)])
chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)])
# Tuning variables
embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],)
class ProcessorForm(FlaskForm):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])
# Catalog for the Retriever
catalog = QuerySelectField(
'Catalog ID',
query_factory=lambda: Catalog.query.all(),
allow_blank=True,
get_label='name',
validators=[Optional()],
)
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = SelectField('Processor Type', validators=[DataRequired()])
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
tuning = BooleanField('Enable Embedding Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Dynamically populate the 'type' field using the constructor
self.type.choices = [(key, value['name']) for key, value in PROCESSOR_TYPES.items()]
class EditProcessorForm(DynamicFormBase):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])
# Catalog for the Retriever
catalog = QuerySelectField(
'Catalog ID',
query_factory=lambda: Catalog.query.all(),
allow_blank=True,
get_label='name',
validators=[Optional()],
)
type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True})
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
tuning = BooleanField('Enable Embedding Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
class RetrieverForm(FlaskForm):
@@ -135,22 +169,17 @@ class EditRetrieverForm(DynamicFormBase):
validators=[Optional()],
)
# Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config)
type = SelectField('Retriever Type', validators=[DataRequired()], render_kw={'readonly': True})
type = StringField('Processor Type', validators=[DataRequired()], render_kw={'readonly': True})
tuning = BooleanField('Enable Tuning', default=False)
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Set the retriever type choices (loaded from config)
self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
class AddDocumentForm(DynamicFormBase):
file = FileField('File', validators=[FileRequired(), allowed_file])
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()])
user_context = TextAreaField('User Context', validators=[Optional()])
@@ -167,6 +196,7 @@ class AddDocumentForm(DynamicFormBase):
class AddURLForm(DynamicFormBase):
url = URLField('URL', validators=[DataRequired(), URL()])
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()])
user_context = TextAreaField('User Context', validators=[Optional()])
@@ -207,6 +237,7 @@ class EditDocumentForm(FlaskForm):
class EditDocumentVersionForm(DynamicFormBase):
sub_file_type = StringField('Sub File Type', validators=[Optional(), Length(max=50)])
language = StringField('Language')
user_context = TextAreaField('User Context', validators=[Optional()])
system_context = TextAreaField('System Context', validators=[Optional()])

172
eveai_app/views/document_views.py
View File

@@ -14,15 +14,16 @@ from urllib.parse import urlparse, unquote
import io
import json
from common.models.document import Document, DocumentVersion, Catalog, Retriever
from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor
from common.extensions import db, minio_client
from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
process_multiple_urls, get_documents_list, edit_document, \
edit_document_version, refresh_document
from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
EveAIDoubleURLException
from config.processor_types import PROCESSOR_TYPES
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
@@ -37,13 +38,11 @@ document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@document_bp.before_request
def log_before_request():
current_app.logger.debug(f"Before request (document_bp): {request.method} {request.url}")
pass
@document_bp.after_request
def log_after_request(response):
current_app.logger.debug(
f"After request (document_bp): {request.method} {request.url} - Status: {response.status}")
return response
@@ -53,8 +52,6 @@ def before_request():
mw_before_request()
except Exception as e:
current_app.logger.error(f'Error switching schema in Document Blueprint: {e}')
for role in current_user.roles:
current_app.logger.debug(f'User {current_user.email} has role {role.name}')
raise
@@ -67,16 +64,6 @@ def catalog():
tenant_id = session.get('tenant').get('id')
new_catalog = Catalog()
form.populate_obj(new_catalog)
# Handle Embedding Variables
new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
if form.html_end_tags.data else []
new_catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
if form.html_included_elements.data else []
new_catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
if form.html_excluded_elements.data else []
new_catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
set_logging_information(new_catalog, dt.now(tz.utc))
try:
@@ -84,6 +71,8 @@ def catalog():
db.session.commit()
flash('Catalog successfully added!', 'success')
current_app.logger.info(f'Catalog {new_catalog.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of catalog - add configuration of the catalog (dependent on type)
return redirect(prefixed_url_for('document_bp.catalog', catalog_id=new_catalog.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add catalog. Error: {e}', 'danger')
@@ -140,27 +129,8 @@ def edit_catalog(catalog_id):
configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
# Convert arrays to comma-separated strings for display
if request.method == 'GET':
form.html_tags.data = ', '.join(catalog.html_tags or '')
form.html_end_tags.data = ', '.join(catalog.html_end_tags or '')
form.html_included_elements.data = ', '.join(catalog.html_included_elements or '')
form.html_excluded_elements.data = ', '.join(catalog.html_excluded_elements or '')
form.html_excluded_classes.data = ', '.join(catalog.html_excluded_classes or '')
if request.method == 'POST' and form.validate_on_submit():
form.populate_obj(catalog)
# Handle Embedding Variables
catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
if form.html_end_tags.data else []
catalog.html_included_elements = [tag.strip() for tag in form.html_included_elements.data.split(',')] \
if form.html_included_elements.data else []
catalog.html_excluded_elements = [tag.strip() for tag in form.html_excluded_elements.data.split(',')] \
if form.html_excluded_elements.data else []
catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
catalog.configuration = form.get_dynamic_data('configuration')
update_logging_information(catalog, dt.now(tz.utc))
try:
@@ -180,6 +150,116 @@ def edit_catalog(catalog_id):
return render_template('document/edit_catalog.html', form=form, catalog_id=catalog_id)
@document_bp.route('/processor', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def processor():
form = ProcessorForm()
if form.validate_on_submit():
tenant_id = session.get('tenant').get('id')
new_processor = Processor()
form.populate_obj(new_processor)
new_processor.catalog_id = form.catalog.data.id
set_logging_information(new_processor, dt.now(tz.utc))
try:
db.session.add(new_processor)
db.session.commit()
flash('Processor successfully added!', 'success')
current_app.logger.info(f'Processor {new_processor.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=new_processor.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add processor. Error: {e}', 'danger')
current_app.logger.error(f'Failed to add retriever {new_processor.name}'
f'for tenant {tenant_id}. Error: {str(e)}')
return render_template('document/processor.html', form=form)
@document_bp.route('/processor/<int:processor_id>', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def edit_processor(processor_id):
"""Edit an existing processorr configuration."""
# Get the processor or return 404
processor = Processor.query.get_or_404(processor_id)
if processor.catalog_id:
# If catalog_id is just an ID, fetch the Catalog object
processor.catalog = Catalog.query.get(processor.catalog_id)
else:
processor.catalog = None
# Create form instance with the processor
form = EditProcessorForm(request.form, obj=processor)
configuration_config = PROCESSOR_TYPES[processor.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, processor.configuration)
if form.validate_on_submit():
# Update basic fields
form.populate_obj(processor)
processor.configuration = form.get_dynamic_data('configuration')
# Update catalog relationship
processor.catalog_id = form.catalog.data.id if form.catalog.data else None
# Update logging information
update_logging_information(processor, dt.now(tz.utc))
# Save changes to database
try:
db.session.add(processor)
db.session.commit()
flash('Retriever updated successfully!', 'success')
current_app.logger.info(f'Retriever {processor.id} updated successfully')
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to update processor. Error: {str(e)}', 'danger')
current_app.logger.error(f'Failed to update processor {processor_id}. Error: {str(e)}')
return render_template('document/edit_processor.html', form=form, processor_id=processor_id)
return redirect(prefixed_url_for('document_bp.processors'))
else:
form_validation_failed(request, form)
return render_template('document/edit_processor.html', form=form, processor_id=processor_id)
@document_bp.route('/processors', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def processors():
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
query = Processor.query.order_by(Processor.id)
pagination = query.paginate(page=page, per_page=per_page)
the_processors = pagination.items
# prepare table data
rows = prepare_table_for_macro(the_processors,
[('id', ''), ('name', ''), ('type', ''), ('catalog_id', '')])
# Render the catalogs in a template
return render_template('document/processors.html', rows=rows, pagination=pagination)
@document_bp.route('/handle_processor_selection', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def handle_processor_selection():
processor_identification = request.form.get('selected_row')
processor_id = ast.literal_eval(processor_identification).get('value')
action = request.form['action']
if action == 'edit_processor':
return redirect(prefixed_url_for('document_bp.edit_processor', processor_id=processor_id))
return redirect(prefixed_url_for('document_bp.processors'))
@document_bp.route('/retriever', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def retriever():
@@ -198,15 +278,14 @@ def retriever():
db.session.commit()
flash('Retriever successfully added!', 'success')
current_app.logger.info(f'Catalog {new_retriever.name} successfully added for tenant {tenant_id}!')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.edit_retriever', retriever_id=new_retriever.id))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to add retriever. Error: {e}', 'danger')
current_app.logger.error(f'Failed to add retriever {new_retriever.name}'
f'for tenant {tenant_id}. Error: {str(e)}')
# Enable step 2 of creation of retriever - add configuration of the retriever (dependent on type)
return redirect(prefixed_url_for('document_bp.retriever', retriever_id=new_retriever.id))
return render_template('document/retriever.html', form=form)
@@ -311,6 +390,7 @@ def add_document():
current_app.logger.info(f'Adding Document for {catalog_id}')
tenant_id = session['tenant']['id']
file = form.file.data
sub_file_type = form.sub_file_type.data
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
@@ -324,14 +404,13 @@ def add_document():
api_input = {
'catalog_id': catalog_id,
'name': form.name.data,
'sub_file_type': form.sub_file_type.data,
'language': form.language.data,
'user_context': form.user_context.data,
'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
'catalog_properties': catalog_properties,
}
current_app.logger.debug(f'Creating document stack with input {api_input}')
new_doc, new_doc_vers = create_document_stack(api_input, file, filename, extension, tenant_id)
task_id = start_embedding_task(tenant_id, new_doc_vers.id)
@@ -341,6 +420,7 @@ def add_document():
except (EveAIInvalidLanguageException, EveAIUnsupportedFileType) as e:
flash(str(e), 'error')
current_app.logger.error(f"Error adding document: {str(e)}")
except Exception as e:
current_app.logger.error(f'Error adding document: {str(e)}')
flash('An error occurred while adding the document.', 'error')
@@ -378,6 +458,7 @@ def add_url():
api_input = {
'catalog_id': catalog_id,
'name': form.name.data or filename,
'sub_file_type': form.sub_file_type.data,
'url': url,
'language': form.language.data,
'user_context': form.user_context.data,
@@ -562,8 +643,6 @@ def handle_document_version_selection():
action = request.form['action']
current_app.logger.debug(f'Triggered Document Version Action: {action}')
match action:
case 'edit_document_version':
return redirect(prefixed_url_for('document_bp.edit_document_version_view', document_version_id=doc_vers_id))
@@ -598,9 +677,7 @@ def handle_library_selection():
@document_bp.route('/document_versions_list', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin')
def document_versions_list():
current_app.logger.debug('Getting document versions list')
view = DocumentVersionListView(DocumentVersion, 'document/document_versions_list_view.html', per_page=20)
current_app.logger.debug('Got document versions list')
return view.get()
@@ -653,8 +730,9 @@ def update_logging_information(obj, timestamp):
def log_session_state(session, msg=""):
current_app.logger.debug(f"{msg} - Session dirty: {session.dirty}")
current_app.logger.debug(f"{msg} - Session new: {session.new}")
pass
# current_app.logger.info(f"{msg} - Session dirty: {session.dirty}")
# current_app.logger.info(f"{msg} - Session new: {session.new}")
def fetch_html(url):

29
eveai_app/views/dynamic_form_base.py
View File

@@ -5,6 +5,7 @@ import json
from wtforms.fields.choices import SelectField
from wtforms.fields.datetime import DateField
from common.utils.config_field_types import TaggingFields
class DynamicFormBase(FlaskForm):
@@ -38,14 +39,35 @@ class DynamicFormBase(FlaskForm):
message=f"Value must be between {min_value or '-∞'} and {max_value or '∞'}"
)
)
elif field_type == 'tagging_fields':
validators_list.append(self._validate_tagging_fields)
return validators_list
def _validate_tagging_fields(self, form, field):
"""Validate the tagging fields structure"""
if not field.data:
return
try:
# Parse JSON data
fields_data = json.loads(field.data)
# Validate using TaggingFields model
try:
TaggingFields.from_dict(fields_data)
except ValueError as e:
raise ValidationError(str(e))
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except Exception as e:
raise ValidationError(f"Invalid field definition: {str(e)}")
def add_dynamic_fields(self, collection_name, config, initial_data=None):
"""Add dynamic fields to the form based on the configuration."""
self.dynamic_fields[collection_name] = []
for field_name, field_def in config.items():
current_app.logger.debug(f"{field_name}: {field_def}")
# Prefix the field name with the collection name
full_field_name = f"{collection_name}_{field_name}"
label = field_def.get('name', field_name)
@@ -59,7 +81,6 @@ class DynamicFormBase(FlaskForm):
# Handle special case for tagging_fields
if field_type == 'tagging_fields':
field_class = TextAreaField
field_validators.append(validate_tagging_fields)
extra_classes = 'json-editor'
field_kwargs = {}
elif field_type == 'enum':
@@ -145,16 +166,12 @@ class DynamicFormBase(FlaskForm):
def get_dynamic_data(self, collection_name):
"""Retrieve the data from dynamic fields of a specific collection."""
data = {}
current_app.logger.debug(f"{collection_name} in {self.dynamic_fields}?")
if collection_name not in self.dynamic_fields:
return data
prefix_length = len(collection_name) + 1 # +1 for the underscore
for full_field_name in self.dynamic_fields[collection_name]:
current_app.logger.debug(f"{full_field_name}: {full_field_name}")
original_field_name = full_field_name[prefix_length:]
current_app.logger.debug(f"{original_field_name}: {original_field_name}")
field = getattr(self, full_field_name)
current_app.logger.debug(f"{field}: {field}")
# Parse JSON for tagging_fields type
if isinstance(field, TextAreaField) and field.data:
try:

1
eveai_app/views/entitlements_views.py
View File

@@ -159,7 +159,6 @@ def create_license(license_tier_id):
tenant_id=tenant_id,
tier_id=license_tier_id,
)
current_app.logger.debug(f"Currency data in form: {form.currency.data}")
if form.validate_on_submit():
# Update the license with form data
form.populate_obj(new_license)

2
eveai_app/views/interaction_forms.py
View File

@@ -33,7 +33,7 @@ class SpecialistForm(FlaskForm):
type = SelectField('Specialist Type', validators=[DataRequired()])
tuning = BooleanField('Enable Retrieval Tuning', default=False)
tuning = BooleanField('Enable Specialist Tuning', default=False)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)

40
eveai_app/views/interaction_views.py
View File

@@ -32,13 +32,11 @@ interaction_bp = Blueprint('interaction_bp', __name__, url_prefix='/interaction'
@interaction_bp.before_request
def log_before_request():
current_app.logger.debug(f"Before request (interaction_bp): {request.method} {request.url}")
pass
@interaction_bp.after_request
def log_after_request(response):
current_app.logger.debug(
f"After request (interaction_bp): {request.method} {request.url} - Status: {response.status}")
return response
@@ -48,8 +46,6 @@ def before_request():
mw_before_request()
except Exception as e:
current_app.logger.error(f'Error switching schema in Interaction Blueprint: {e}')
for role in current_user.roles:
current_app.logger.debug(f'User {current_user.email} has role {role.name}')
raise
@@ -147,14 +143,9 @@ def specialist():
db.session.add(new_specialist)
db.session.flush() # This assigns the ID to the specialist without committing the transaction
current_app.logger.debug(
f'New Specialist after flush - id: {new_specialist.id}, name: {new_specialist.name}')
# Create the retriever associations
selected_retrievers = form.retrievers.data
current_app.logger.debug(f'Selected Retrievers - {selected_retrievers}')
for retriever in selected_retrievers:
current_app.logger.debug(f'Creating association for Retriever - {retriever.id}')
specialist_retriever = SpecialistRetriever(
specialist_id=new_specialist.id,
retriever_id=retriever.id
@@ -174,7 +165,7 @@ def specialist():
flash(f'Failed to add specialist. Error: {str(e)}', 'danger')
return render_template('interaction/specialist.html', form=form)
return render_template('interaction/specialists.html', form=form)
return render_template('interaction/specialist.html', form=form)
@interaction_bp.route('/specialist/<int:specialist_id>', methods=['GET', 'POST'])
@@ -187,35 +178,31 @@ def edit_specialist(specialist_id):
form.add_dynamic_fields("configuration", configuration_config, specialist.configuration)
if request.method == 'GET':
# Pre-populate the retrievers field with current associations
current_app.logger.debug(f'Specialist retrievers: {specialist.retrievers}')
current_app.logger.debug(f'Form Retrievers Data Before: {form.retrievers.data}')
# Get the actual Retriever objects for the associated retriever_ids
retriever_objects = Retriever.query.filter(
Retriever.id.in_([sr.retriever_id for sr in specialist.retrievers])
).all()
form.retrievers.data = retriever_objects
current_app.logger.debug(f'Form Retrievers Data After: {form.retrievers.data}')
if form.validate_on_submit():
# Update the basic fields
form.populate_obj(specialist)
specialist.name = form.name.data
specialist.description = form.description.data
specialist.tuning = form.tuning.data
# Update the configuration dynamic fields
specialist.configuration = form.get_dynamic_data("configuration")
# Update retriever associations
current_retrievers = set(sr.retriever_id for sr in specialist.retrievers)
selected_retrievers = set(r.id for r in form.retrievers.data)
# Get current and selected retrievers
current_retrievers = {sr.retriever_id: sr for sr in specialist.retrievers}
selected_retrievers = {r.id: r for r in form.retrievers.data}
# Remove unselected retrievers
for sr in specialist.retrievers[:]:
if sr.retriever_id not in selected_retrievers:
db.session.delete(sr)
for retriever_id in set(current_retrievers.keys()) - set(selected_retrievers.keys()):
specialist_retriever = current_retrievers[retriever_id]
db.session.delete(specialist_retriever)
# Add new retrievers
for retriever_id in selected_retrievers - current_retrievers:
for retriever_id in set(selected_retrievers.keys()) - set(current_retrievers.keys()):
specialist_retriever = SpecialistRetriever(
specialist_id=specialist.id,
retriever_id=retriever_id
@@ -229,13 +216,12 @@ def edit_specialist(specialist_id):
db.session.commit()
flash('Specialist updated successfully!', 'success')
current_app.logger.info(f'Specialist {specialist.id} updated successfully')
return redirect(prefixed_url_for('interaction_bp.specialists'))
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to update specialist. Error: {str(e)}', 'danger')
current_app.logger.error(f'Failed to update specialist {specialist_id}. Error: {str(e)}')
return render_template('interaction/edit_specialist.html', form=form, specialist_id=specialist_id)
return redirect(prefixed_url_for('interaction_bp.specialists'))
else:
form_validation_failed(request, form)

31
eveai_app/views/security_views.py
View File

@@ -22,20 +22,11 @@ security_bp = Blueprint('security_bp', __name__)
@security_bp.before_request
def log_before_request():
current_app.logger.debug(f"Before request (security_bp): {request.method} {request.url}")
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (security_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (security_bp): No user logged in")
pass
@security_bp.after_request
def log_after_request(response):
current_app.logger.debug(f"After request (security_bp): {request.method} {request.url} - Status: {response.status}")
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (security_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (security_bp): No user logged in")
return response
@@ -47,13 +38,12 @@ def login():
form = LoginForm()
if request.method == 'POST':
current_app.logger.debug(f"Starting login procedure for {form.email.data}")
try:
if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data).first()
if user is None or not verify_and_update_password(form.password.data, user):
flash('Invalid username or password', 'danger')
current_app.logger.debug(f'Failed to login user')
current_app.logger.error(f'Failed to login user')
return redirect(prefixed_url_for('security_bp.login'))
if login_user(user):
@@ -65,10 +55,10 @@ def login():
return redirect(prefixed_url_for('user_bp.tenant_overview'))
else:
flash('Invalid username or password', 'danger')
current_app.logger.debug(f'Failed to login user {user.email}')
current_app.logger.error(f'Failed to login user {user.email}')
abort(401)
else:
current_app.logger.debug(f'Invalid login form: {form.errors}')
current_app.logger.error(f'Invalid login form: {form.errors}')
except CSRFError:
current_app.logger.warning('CSRF token mismatch during login attempt')
@@ -77,19 +67,14 @@ def login():
if request.method == 'GET':
csrf_token = generate_csrf()
current_app.logger.debug(f'Generated new CSRF token: {csrf_token}')
# current_app.logger.debug(f"Login route completed - Session ID: {session.sid}")
current_app.logger.debug(f"Login route completed - Session data: {session}")
return render_template('security/login_user.html', login_user_form=form)
@security_bp.route('/logout', methods=['GET', 'POST'])
@login_required
def logout():
current_app.logger.debug('Logging out')
logout_user()
current_app.logger.debug('After Logout')
return redirect(prefixed_url_for('basic_bp.index'))
@@ -99,17 +84,13 @@ def confirm_email(token):
email = confirm_token(token)
except Exception as e:
flash('The confirmation link is invalid or has expired.', 'danger')
current_app.logger.debug(f'Invalid confirmation link detected: {token} - error: {e}')
return redirect(prefixed_url_for('basic_bp.confirm_email_fail'))
user = User.query.filter_by(email=email).first_or_404()
current_app.logger.debug(f'Trying to confirm email for user {user.email}')
if user.active:
flash('Account already confirmed. Please login.', 'success')
current_app.logger.debug(f'Account for user {user.email} was already activated')
return redirect(prefixed_url_for('security_bp.login'))
else:
current_app.logger.debug(f'Trying to confirm email for user {user.email}')
user.active = True
user.updated_at = dt.now(tz.utc)
user.confirmed_at = dt.now(tz.utc)
@@ -119,10 +100,8 @@ def confirm_email(token):
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.debug(f'Failed to confirm email for user {user.email}: {e}')
return redirect(prefixed_url_for('basic_bp.confirm_email_fail'))
current_app.logger.debug(f'Account for user {user.email} was confirmed.')
send_reset_email(user)
return redirect(prefixed_url_for('basic_bp.confirm_email_ok'))
@@ -145,7 +124,7 @@ def reset_password(token):
email = confirm_token(token)
except Exception as e:
flash('The reset link is invalid or has expired.', 'danger')
current_app.logger.debug(f'Invalid reset link detected: {token} - error: {e}')
current_app.logger.error(f'Invalid reset link detected: {token} - error: {e}')
return redirect(prefixed_url_for('security_bp.reset_password_request'))
user = User.query.filter_by(email=email).first_or_404()

20
eveai_app/views/user_views.py
View File

@@ -21,20 +21,11 @@ user_bp = Blueprint('user_bp', __name__, url_prefix='/user')
@user_bp.before_request
def log_before_request():
current_app.logger.debug(f"Before request (user_bp): {request.method} {request.url}")
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (user_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (user_bp): No user logged in")
pass
@user_bp.after_request
def log_after_request(response):
current_app.logger.debug(f"After request (user_bp): {request.method} {request.url} - Status: {response.status}")
if current_user and current_user.is_authenticated:
current_app.logger.debug(f"After request (user_bp): Current User: {current_user.email}")
else:
current_app.logger.debug(f"After request (user_bp): No user logged in")
return response
@@ -43,7 +34,6 @@ def log_after_request(response):
def tenant():
form = TenantForm()
if form.validate_on_submit():
current_app.logger.debug('Creating new tenant')
# Handle the required attributes
new_tenant = Tenant()
form.populate_obj(new_tenant)
@@ -91,7 +81,6 @@ def edit_tenant(tenant_id):
form.populate_obj(tenant)
if form.validate_on_submit():
current_app.logger.debug(f'Updating tenant {tenant_id}')
# Populate the tenant with form data
form.populate_obj(tenant)
@@ -102,7 +91,6 @@ def edit_tenant(tenant_id):
session['tenant'] = tenant.to_dict()
# return redirect(url_for(f"user/tenant/tenant_id"))
else:
current_app.logger.debug(f'Tenant update failed with errors: {form.errors}')
form_validation_failed(request, form)
return render_template('user/tenant.html', form=form, tenant_id=tenant_id)
@@ -142,7 +130,7 @@ def user():
# security.datastore.set_uniquifier(new_user)
try:
send_confirmation_email(new_user)
current_app.logger.debug(f'User {new_user.id} with name {new_user.user_name} added to database'
current_app.logger.info(f'User {new_user.id} with name {new_user.user_name} added to database'
f'Confirmation email sent to {new_user.email}')
flash('User added successfully and confirmation email sent.', 'success')
except Exception as e:
@@ -448,11 +436,7 @@ def generate_api_api_key():
@user_bp.route('/tenant_overview', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin')
def tenant_overview():
current_app.logger.debug('Rendering tenant overview')
current_app.logger.debug(f'current_user: {current_user}')
current_app.logger.debug(f'Current user roles: {current_user.roles}')
tenant_id = session['tenant']['id']
current_app.logger.debug(f'Generating overview for tenant {tenant_id}')
tenant = Tenant.query.get_or_404(tenant_id)
form = TenantForm(obj=tenant)
return render_template('user/tenant_overview.html', form=form)