pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Add dynamic fields to DocumentVersion in case the Catalog requires it.

Browse Source
This commit is contained in:
Josako
2024-10-30 13:52:18 +01:00
parent 43547287b1
commit 532073d38e
11 changed files with 146 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
eveai_app/templates/document/add_document.html
View File

@@ -11,9 +11,17 @@
{{ form.hidden_tag() }}
{% set disabled_fields = [] %}
{% set exclude_fields = [] %}
{% for field in form %}
{% for field in form.get_static_fields() %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
{% for collection_name, fields in form.get_dynamic_fields().items() %}
{% if fields|length > 0 %}
<h4 class="mt-4">{{ collection_name }}</h4>
{% endif %}
{% for field in fields %}
{{ render_field(field, disabled_fields, exclude_fields) }}
{% endfor %}
{% endfor %}
<button type="submit" class="btn btn-primary">Add Document</button>
</form>
{% endblock %}

18
eveai_app/views/document_forms.py
View File

@@ -149,7 +149,7 @@ class EditRetrieverForm(DynamicFormBase):
self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
class AddDocumentForm(FlaskForm):
class AddDocumentForm(DynamicFormBase):
file = FileField('File', validators=[FileRequired(), allowed_file])
name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()])
@@ -157,17 +157,15 @@ class AddDocumentForm(FlaskForm):
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
submit = SubmitField('Submit')
def __init__(self):
super().__init__()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.language.choices = [(language, language) for language in
session.get('tenant').get('allowed_languages')]
if not self.language.data:
self.language.data = session.get('tenant').get('default_language')
class AddURLForm(FlaskForm):
class AddURLForm(DynamicFormBase):
url = URLField('URL', validators=[DataRequired(), URL()])
name = StringField('Name', validators=[Length(max=100)])
language = SelectField('Language', choices=[], validators=[Optional()])
@@ -175,10 +173,8 @@ class AddURLForm(FlaskForm):
valid_from = DateField('Valid from', id='form-control datepicker', validators=[Optional()])
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
submit = SubmitField('Submit')
def __init__(self):
super().__init__()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.language.choices = [(language, language) for language in
session.get('tenant').get('allowed_languages')]
if not self.language.data:
@@ -210,7 +206,7 @@ class EditDocumentForm(FlaskForm):
submit = SubmitField('Submit')
class EditDocumentVersionForm(FlaskForm):
class EditDocumentVersionForm(DynamicFormBase):
language = StringField('Language')
user_context = TextAreaField('User Context', validators=[Optional()])
system_context = TextAreaField('System Context', validators=[Optional()])

100
eveai_app/views/document_views.py
View File

@@ -228,6 +228,8 @@ def edit_retriever(retriever_id):
configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, retriever.configuration)
if request.method == 'POST':
current_app.logger.debug(f'Received POST request with {request.form}')
if form.validate_on_submit():
# Update basic fields
@@ -295,19 +297,33 @@ def handle_retriever_selection():
@document_bp.route('/add_document', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def add_document():
form = AddDocumentForm()
form = AddDocumentForm(request.form)
catalog_id = session.get('catalog_id', None)
if catalog_id is None:
flash('You need to set a Session Catalog before adding Documents or URLs')
return redirect(prefixed_url_for('document_bp.catalogs'))
catalog = Catalog.query.get_or_404(catalog_id)
if catalog.configuration and len(catalog.configuration) > 0:
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
for config in document_version_configurations:
form.add_dynamic_fields(config, catalog.configuration[config])
if form.validate_on_submit():
try:
current_app.logger.info(f'Adding Document for {catalog_id}')
tenant_id = session['tenant']['id']
catalog_id = session['catalog_id']
file = form.file.data
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
validate_file_type(extension)
current_app.logger.debug(f'Language on form: {form.language.data}')
catalog_properties = {}
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
for config in document_version_configurations:
catalog_properties[config] = form.get_dynamic_data(config)
api_input = {
'catalog_id': catalog_id,
'name': form.name.data,
@@ -315,6 +331,7 @@ def add_document():
'user_context': form.user_context.data,
'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
'catalog_properties': catalog_properties,
}
current_app.logger.debug(f'Creating document stack with input {api_input}')
@@ -337,16 +354,30 @@ def add_document():
@document_bp.route('/add_url', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def add_url():
form = AddURLForm()
form = AddURLForm(request.form)
catalog_id = session.get('catalog_id', None)
if catalog_id is None:
flash('You need to set a Session Catalog before adding Documents or URLs')
return redirect(prefixed_url_for('document_bp.catalogs'))
catalog = Catalog.query.get_or_404(catalog_id)
if catalog.configuration and len(catalog.configuration) > 0:
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
for config in document_version_configurations:
form.add_dynamic_fields(config, catalog.configuration[config])
if form.validate_on_submit():
try:
tenant_id = session['tenant']['id']
catalog_id = session['catalog_id']
url = form.url.data
file_content, filename, extension = process_url(url, tenant_id)
catalog_properties = {}
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
for config in document_version_configurations:
catalog_properties[config] = form.get_dynamic_data(config)
api_input = {
'catalog_id': catalog_id,
'name': form.name.data or filename,
@@ -355,6 +386,7 @@ def add_url():
'user_context': form.user_context.data,
'valid_from': form.valid_from.data,
'user_metadata': json.loads(form.user_metadata.data) if form.user_metadata.data else None,
'catalog_properties': catalog_properties,
}
new_doc, new_doc_vers = create_document_stack(api_input, file_content, filename, extension, tenant_id)
@@ -375,43 +407,6 @@ def add_url():
return render_template('document/add_url.html', form=form)
@document_bp.route('/add_urls', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def add_urls():
form = AddURLsForm()
if form.validate_on_submit():
try:
tenant_id = session['tenant']['id']
urls = form.urls.data.split('\n')
urls = [url.strip() for url in urls if url.strip()]
api_input = {
'name': form.name.data,
'language': form.language.data,
'user_context': form.user_context.data,
'valid_from': form.valid_from.data
}
results = process_multiple_urls(urls, tenant_id, api_input)
for result in results:
if result['status'] == 'success':
flash(
f"Processed URL: {result['url']} - Document ID: {result['document_id']}, Version ID: {result['document_version_id']}",
'success')
else:
flash(f"Error processing URL: {result['url']} - {result['message']}", 'error')
return redirect(prefixed_url_for('document_bp.documents'))
except Exception as e:
current_app.logger.error(f'Error adding multiple URLs: {str(e)}')
flash('An error occurred while adding the URLs.', 'error')
return render_template('document/add_urls.html', form=form)
@document_bp.route('/documents', methods=['GET', 'POST'])
@roles_accepted('Super User', 'Tenant Admin')
def documents():
@@ -494,12 +489,29 @@ def edit_document_view(document_id):
@roles_accepted('Super User', 'Tenant Admin')
def edit_document_version_view(document_version_id):
doc_vers = DocumentVersion.query.get_or_404(document_version_id)
form = EditDocumentVersionForm(obj=doc_vers)
form = EditDocumentVersionForm(request.form, obj=doc_vers)
catalog_id = session.get('catalog_id', None)
if catalog_id is None:
flash('You need to set a Session Catalog before adding Documents or URLs')
return redirect(prefixed_url_for('document_bp.catalogs'))
catalog = Catalog.query.get_or_404(catalog_id)
if catalog.configuration and len(catalog.configuration) > 0:
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
for config in document_version_configurations:
form.add_dynamic_fields(config, catalog.configuration[config], doc_vers.catalog_properties[config])
if form.validate_on_submit():
catalog_properties = {}
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
for config in document_version_configurations:
catalog_properties[config] = form.get_dynamic_data(config)
updated_version, error = edit_document_version(
document_version_id,
form.user_context.data
form.user_context.data,
catalog_properties,
)
if updated_version:
flash(f'Document Version {updated_version.id} updated successfully', 'success')

16
eveai_app/views/dynamic_form_base.py
View File

@@ -48,10 +48,9 @@ class DynamicFormBase(FlaskForm):
current_app.logger.debug(f"{field_name}: {field_def}")
# Prefix the field name with the collection name
full_field_name = f"{collection_name}_{field_name}"
label = field_def.get('name')
label = field_def.get('name', field_name)
field_type = field_def.get('type')
description = field_def.get('description', '')
required = field_def.get('required', False)
default = field_def.get('default')
# Determine standard validators
@@ -90,8 +89,8 @@ class DynamicFormBase(FlaskForm):
except (TypeError, ValueError) as e:
current_app.logger.error(f"Error converting initial data to JSON: {e}")
field_data = "{}"
elif field_def.get('default') is not None:
field_data = field_def.get('default')
elif default is not None:
field_data = default
# Create render_kw with classes and any other HTML attributes
render_kw = {'class': extra_classes} if extra_classes else {}
@@ -145,12 +144,16 @@ class DynamicFormBase(FlaskForm):
def get_dynamic_data(self, collection_name):
"""Retrieve the data from dynamic fields of a specific collection."""
data = {}
current_app.logger.debug(f"{collection_name} in {self.dynamic_fields}?")
if collection_name not in self.dynamic_fields:
return data
prefix_length = len(collection_name) + 1 # +1 for the underscore
for full_field_name in self.dynamic_fields[collection_name]:
current_app.logger.debug(f"{full_field_name}: {full_field_name}")
original_field_name = full_field_name[prefix_length:]
current_app.logger.debug(f"{original_field_name}: {original_field_name}")
field = getattr(self, full_field_name)
current_app.logger.debug(f"{field}: {field}")
# Parse JSON for tagging_fields type
if isinstance(field, TextAreaField) and field.data:
try:
@@ -207,4 +210,7 @@ def validate_tagging_fields(form, field):
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except (TypeError, ValueError) as e:
raise ValidationError(f"Invalid field definition: {str(e)}")
raise ValidationError(f"Invalid field definition: {str(e)}")