pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Created a new eveai_chat plugin to support the new dynamic possibilities of the Specialists. Currently only supports standard Rag retrievers (i.e. no extra arguments).

Browse Source
This commit is contained in:
Josako
2024-11-27 12:26:49 +01:00
parent 07d89d204f
commit 98cb4e4f2f
13 changed files with 462 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
eveai_app/views/document_views.py
View File

@@ -13,6 +13,7 @@ import json
from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor
from common.extensions import db
from common.models.interaction import Specialist, SpecialistRetriever
from common.utils.document_utils import validate_file_type, create_document_stack, start_embedding_task, process_url, \
edit_document, \
edit_document_version, refresh_document
@@ -663,6 +664,8 @@ def handle_library_selection():
action = request.form['action']
match action:
case 'create_default_rag_library':
create_default_rag_library()
case 're_embed_latest_versions':
re_embed_latest_versions()
case 'refresh_all_documents':
@@ -671,6 +674,86 @@ def handle_library_selection():
return redirect(prefixed_url_for('document_bp.library_operations'))
def create_default_rag_library():
# Check if no catalog exists. If non exists, no processors, retrievers or specialist can exists
catalogs = Catalog.query.all()
if catalogs:
flash("Default RAG Library can only be created if no catalogs are defined!", 'danger')
return redirect(prefixed_url_for('document_bp.library_operations'))
timestamp = dt.now(tz=tz.utc)
try:
cat = Catalog(
name='Default RAG Catalog',
description='Default RAG Catalog',
type="STANDARD_CATALOG",
min_chunk_size=2000,
max_chunk_size=3000,
)
set_logging_information(cat, timestamp)
db.session.add(cat)
db.session.commit()
proc = Processor(
name='Default HTML Processor',
description='Default HTML Processor',
catalog_id=cat.id,
type="HTML Processor",
configuration={
"html_tags": "p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td",
"html_end_tags": "p, li, table",
"html_excluded_classes": "",
"html_excluded_elements": "header, footer, nav, script",
"html_included_elements": "article, main"
}
)
set_logging_information(proc, timestamp)
retr = Retriever(
name='Default RAG Retriever',
description='Default RAG Retriever',
catalog_id=cat.id,
type="STANDARD_RAG",
configuration={
"es_k": "8",
"es_similarity_threshold": 0.3
}
)
set_logging_information(retr, timestamp)
db.session.add(proc)
db.session.add(retr)
db.session.commit()
spec = Specialist(
name='Default RAG Specialist',
description='Default RAG Specialist',
type='STANDARD_RAG',
configuration={"temperature": "0.3", "specialist_context": "To be specified"}
)
set_logging_information(spec, timestamp)
db.session.add(spec)
db.session.commit()
spec_retr = SpecialistRetriever(
specialist_id=spec.id,
retriever_id=retr.id,
)
db.session.add(spec_retr)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
flash(f'Failed to create Default RAG Library. Error: {e}', 'danger')
current_app.logger.error(f'Failed to create Default RAG Library'
f'for tenant {session['tenant']['id']}. Error: {str(e)}')
return redirect(prefixed_url_for('document_bp.library_operations'))
@document_bp.route('/document_versions_list', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin')
def document_versions_list():

2
eveai_app/views/entitlements_forms.py
View File

@@ -48,7 +48,7 @@ class LicenseForm(FlaskForm):
start_date = DateField('Start Date', id='form-control datepicker', validators=[DataRequired()])
end_date = DateField('End Date', id='form-control datepicker', validators=[DataRequired()])
currency = StringField('Currency', validators=[Optional(), Length(max=20)])
yearly_payment = BooleanField('Yearly Payment', validators=[DataRequired()], default=False)
yearly_payment = BooleanField('Yearly Payment', default=False)
basic_fee = FloatField('Basic Fee', validators=[InputRequired(), NumberRange(min=0)])
max_storage_mb = IntegerField('Max Storage (MiB)', validators=[DataRequired(), NumberRange(min=1)])
additional_storage_price = FloatField('Additional Storage Token Fee',

20
eveai_app/views/entitlements_views.py
View File

@@ -204,12 +204,11 @@ def edit_license(license_id):
flash('License updated successfully.', 'success')
return redirect(
prefixed_url_for('entitlements_bp.edit_license', license_tier_id=license_id))
prefixed_url_for('entitlements_bp.edit_license', license_id=license_id))
else:
form_validation_failed(request, form)
return render_template('entitlements/license.html', form=form, license_tier_id=license_tier.id,
ext_disabled_fields=disabled_fields)
return render_template('entitlements/license.html', form=form, ext_disabled_fields=disabled_fields)
@entitlements_bp.route('/view_usages')
@@ -258,13 +257,14 @@ def view_licenses():
# Query licenses for the tenant, with ordering and active status
query = (
License.query.filter_by(tenant_id=tenant_id)
License.query
.join(LicenseTier) # Join with LicenseTier
.filter(License.tenant_id == tenant_id)
.add_columns(
License.id,
License.start_date,
License.end_date,
License.license_tier,
License.license_tier.name.label('license_tier_name'),
LicenseTier.name.label('license_tier_name'), # Access name through LicenseTier
((License.start_date <= current_date) &
(or_(License.end_date.is_(None), License.end_date >= current_date))).label('active')
)
@@ -276,7 +276,7 @@ def view_licenses():
# prepare table data
rows = prepare_table_for_macro(lics, [('id', ''), ('license_tier_name', ''), ('start_date', ''), ('end_date', ''),
('active', ''),])
('active', '')])
# Render the licenses in a template
return render_template('entitlements/view_licenses.html', rows=rows, pagination=pagination)
@@ -287,8 +287,10 @@ def view_licenses():
def handle_license_selection():
license_identification = request.form['selected_row']
license_id = ast.literal_eval(license_identification).get('value')
the_license = LicenseUsage.query.get_or_404(license_id)
the_license = License.query.get_or_404(license_id)
action = request.form['action']
pass # Currently, no actions are defined
match action:
case 'edit_license':
return redirect(prefixed_url_for('entitlements_bp.edit_license', license_id=license_id))

16
eveai_app/views/interaction_views.py
View File

@@ -72,11 +72,21 @@ def handle_chat_session_selection():
@interaction_bp.route('/view_chat_session/<int:chat_session_id>', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin')
def view_chat_session(chat_session_id):
# Get chat session with user info
chat_session = ChatSession.query.get_or_404(chat_session_id)
# Get interactions with specialist info
interactions = (Interaction.query
.filter_by(chat_session_id=chat_session.id)
.order_by(Interaction.question_at)
.all())
.join(Specialist, Interaction.specialist_id == Specialist.id, isouter=True)
.add_columns(
Interaction.id,
Interaction.question_at,
Interaction.specialist_arguments,
Interaction.specialist_results,
Specialist.name.label('specialist_name'),
Specialist.type.label('specialist_type')
).order_by(Interaction.question_at).all())
# Fetch all related embeddings for the interactions in this session
embedding_query = (db.session.query(InteractionEmbedding.interaction_id,
@@ -84,7 +94,7 @@ def view_chat_session(chat_session_id):
DocumentVersion.object_name)
.join(Embedding, InteractionEmbedding.embedding_id == Embedding.id)
.join(DocumentVersion, Embedding.doc_vers_id == DocumentVersion.id)
.filter(InteractionEmbedding.interaction_id.in_([i.id for i in interactions])))
.filter(InteractionEmbedding.interaction_id.in_([i.id for i, *_ in interactions])))
# Create a dictionary to store embeddings for each interaction
embeddings_dict = {}

62
eveai_app/views/user_views.py
View File

@@ -68,7 +68,7 @@ def tenant():
current_app.logger.info(f"Creating MinIO bucket for tenant {new_tenant.id}")
minio_client.create_tenant_bucket(new_tenant.id)
return redirect(prefixed_url_for('basic_bp.index'))
return redirect(prefixed_url_for('user_bp.select_tenant'))
else:
form_validation_failed(request, form)
@@ -378,66 +378,6 @@ def edit_tenant_domain(tenant_domain_id):
return render_template('user/edit_tenant_domain.html', form=form, tenant_domain_id=tenant_domain_id)
@user_bp.route('/check_chat_api_key', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def check_chat_api_key():
tenant_id = session['tenant']['id']
tenant = Tenant.query.get_or_404(tenant_id)
if tenant.encrypted_chat_api_key:
return jsonify({'api_key_exists': True})
return jsonify({'api_key_exists': False})
@user_bp.route('/generate_chat_api_key', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def generate_chat_api_key():
tenant = Tenant.query.get_or_404(session['tenant']['id'])
new_api_key = generate_api_key(prefix="EveAI-CHAT")
tenant.encrypted_chat_api_key = simple_encryption.encrypt_api_key(new_api_key)
update_logging_information(tenant, dt.now(tz.utc))
try:
db.session.add(tenant)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.error(f'Unable to store chat api key for tenant {tenant.id}. Error: {str(e)}')
return jsonify({'api_key': new_api_key}), 200
@user_bp.route('/check_api_api_key', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def check_api_api_key():
tenant_id = session['tenant']['id']
tenant = Tenant.query.get_or_404(tenant_id)
if tenant.encrypted_api_key:
return jsonify({'api_key_exists': True})
return jsonify({'api_key_exists': False})
@user_bp.route('/generate_api_api_key', methods=['POST'])
@roles_accepted('Super User', 'Tenant Admin')
def generate_api_api_key():
tenant = Tenant.query.get_or_404(session['tenant']['id'])
new_api_key = generate_api_key(prefix="EveAI-API")
tenant.encrypted_api_key = simple_encryption.encrypt_api_key(new_api_key)
update_logging_information(tenant, dt.now(tz.utc))
try:
db.session.add(tenant)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.error(f'Unable to store api key for tenant {tenant.id}. Error: {str(e)}')
return jsonify({'api_key': new_api_key}), 200
@user_bp.route('/tenant_overview', methods=['GET'])
@roles_accepted('Super User', 'Tenant Admin')
def tenant_overview():