pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Refining & Enhancing dynamic fields

Browse Source 
- Creating a specialized Form class for handling dynamic fields
- Refinement of HTML-macros to handle dynamic fields
- Introduction of dynamic fields for Catalogs
This commit is contained in:
Josako
2024-10-29 09:17:44 +01:00
parent aa358df28e
commit 43547287b1
13 changed files with 605 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
eveai_app/views/document_forms.py
View File

@@ -8,6 +8,7 @@ import json
from wtforms_sqlalchemy.fields import QuerySelectField
from common.extensions import db
from common.models.document import Catalog
from config.catalog_types import CATALOG_TYPES
@@ -34,14 +35,6 @@ def validate_json(form, field):
class CatalogForm(FlaskForm):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])
# Parent ID (Optional for root-level catalogs)
parent = QuerySelectField(
'Parent Catalog',
query_factory=lambda: Catalog.query.all(),
allow_blank=True,
get_label='name',
validators=[Optional()],
)
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = SelectField('Catalog Type', validators=[DataRequired()])
@@ -56,8 +49,9 @@ class CatalogForm(FlaskForm):
default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
default='p, li')
html_included_elements = StringField('HTML Included Elements', validators=[Optional()])
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()])
html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
default='header, footer, nav, script')
html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
@@ -75,6 +69,37 @@ class CatalogForm(FlaskForm):
self.type.choices = [(key, value['name']) for key, value in CATALOG_TYPES.items()]
class EditCatalogForm(DynamicFormBase):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])
# Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
type = StringField('Catalog Type', validators=[DataRequired()], render_kw={'readonly': True})
# Metadata fields
user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],)
# HTML Embedding Variables
html_tags = StringField('HTML Tags', validators=[DataRequired()],
default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
default='p, li')
html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
default='header, footer, nav, script')
html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
default=2000)
max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
default=3000)
# Chat Variables
chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)])
chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)])
# Tuning variables
embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
class RetrieverForm(FlaskForm):
name = StringField('Name', validators=[DataRequired(), Length(max=50)])
description = TextAreaField('Description', validators=[Optional()])

61
eveai_app/views/document_views.py
View File

@@ -22,14 +22,14 @@ from common.utils.document_utils import validate_file_type, create_document_stac
from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
EveAIDoubleURLException
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
CatalogForm, RetrieverForm, EditRetrieverForm
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm
from common.utils.middleware import mw_before_request
from common.utils.celery_utils import current_celery
from common.utils.nginx_utils import prefixed_url_for
from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro, form_to_dict
from .document_list_view import DocumentListView
from .document_version_list_view import DocumentVersionListView
from config.catalog_types import CATALOG_TYPES
from config.retriever_types import RETRIEVER_TYPES
document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -67,7 +67,6 @@ def catalog():
tenant_id = session.get('tenant').get('id')
new_catalog = Catalog()
form.populate_obj(new_catalog)
new_catalog.parent_id = form.parent.data.get('id')
# Handle Embedding Variables
new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
@@ -135,9 +134,12 @@ def handle_catalog_selection():
@roles_accepted('Super User', 'Tenant Admin')
def edit_catalog(catalog_id):
catalog = Catalog.query.get_or_404(catalog_id)
form = CatalogForm(obj=catalog)
tenant_id = session.get('tenant').get('id')
form = EditCatalogForm(request.form, obj=catalog)
configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
# Convert arrays to comma-separated strings for display
if request.method == 'GET':
form.html_tags.data = ', '.join(catalog.html_tags or '')
@@ -158,6 +160,8 @@ def edit_catalog(catalog_id):
if form.html_excluded_elements.data else []
catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
if form.html_excluded_classes.data else []
catalog.configuration = form.get_dynamic_data('configuration')
update_logging_information(catalog, dt.now(tz.utc))
try:
db.session.add(catalog)
@@ -210,8 +214,6 @@ def retriever():
@roles_accepted('Super User', 'Tenant Admin')
def edit_retriever(retriever_id):
"""Edit an existing retriever configuration."""
current_app.logger.debug(f"Editing Retriever {retriever_id}")
# Get the retriever or return 404
retriever = Retriever.query.get_or_404(retriever_id)
@@ -225,7 +227,6 @@ def edit_retriever(retriever_id):
form = EditRetrieverForm(request.form, obj=retriever)
configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"]
current_app.logger.debug(f"Configuration {configuration_config}")
form.add_dynamic_fields("configuration", configuration_config, retriever.configuration)
if form.validate_on_submit():
@@ -663,49 +664,3 @@ def fetch_html(url):
response.raise_for_status() # Will raise an exception for bad requests
return response.content
# def prepare_document_data(docs):
# rows = []
# for doc in docs:
# doc_row = [{'value': doc.name, 'class': '', 'type': 'text'},
# {'value': doc.created_at.strftime("%Y-%m-%d %H:%M:%S"), 'class': '', 'type': 'text'}]
# # Document basic details
# if doc.valid_from:
# doc_row.append({'value': doc.valid_from.strftime("%Y-%m-%d"), 'class': '', 'type': 'text'})
# else:
# doc_row.append({'value': '', 'class': '', 'type': 'text'})
#
# # Nested languages and versions
# languages_rows = []
# for lang in doc.languages:
# lang_row = [{'value': lang.language, 'class': '', 'type': 'text'}]
#
# # Latest version details if available (should be available ;-) )
# if lang.latest_version:
# lang_row.append({'value': lang.latest_version.created_at.strftime("%Y-%m-%d %H:%M:%S"),
# 'class': '', 'type': 'text'})
# if lang.latest_version.url:
# lang_row.append({'value': lang.latest_version.url,
# 'class': '', 'type': 'link', 'href': lang.latest_version.url})
# else:
# lang_row.append({'value': '', 'class': '', 'type': 'text'})
#
# if lang.latest_version.object_name:
# lang_row.append({'value': lang.latest_version.object_name, 'class': '', 'type': 'text'})
# else:
# lang_row.append({'value': '', 'class': '', 'type': 'text'})
#
# if lang.latest_version.file_type:
# lang_row.append({'value': lang.latest_version.file_type, 'class': '', 'type': 'text'})
# else:
# lang_row.append({'value': '', 'class': '', 'type': 'text'})
# # Include other details as necessary
#
# languages_rows.append(lang_row)
#
# doc_row.append({'is_group': True, 'colspan': '5',
# 'headers': ['Language', 'Latest Version', 'URL', 'File Name', 'Type'],
# 'sub_rows': languages_rows})
# rows.append(doc_row)
# return rows

170
eveai_app/views/dynamic_form_base.py
View File

@@ -1,6 +1,11 @@
from flask_wtf import FlaskForm
from wtforms import IntegerField, FloatField, BooleanField, StringField, validators
from wtforms import IntegerField, FloatField, BooleanField, StringField, TextAreaField, validators, ValidationError
from flask import current_app
import json
from wtforms.fields.choices import SelectField
from wtforms.fields.datetime import DateField
class DynamicFormBase(FlaskForm):
def __init__(self, formdata=None, *args, **kwargs):
@@ -10,6 +15,32 @@ class DynamicFormBase(FlaskForm):
# Store formdata for later use
self.formdata = formdata
def _create_field_validators(self, field_def):
"""Create validators based on field definition"""
validators_list = []
# Required validator
if field_def.get('required', False):
validators_list.append(validators.InputRequired())
else:
validators_list.append(validators.Optional())
# Type-specific validators
field_type = field_def.get('type')
if field_type in ['integer', 'float']:
min_value = field_def.get('min_value')
max_value = field_def.get('max_value')
if min_value is not None or max_value is not None:
validators_list.append(
validators.NumberRange(
min=min_value if min_value is not None else -float('inf'),
max=max_value if max_value is not None else float('inf'),
message=f"Value must be between {min_value or '-∞'} and {max_value or '∞'}"
)
)
return validators_list
def add_dynamic_fields(self, collection_name, config, initial_data=None):
"""Add dynamic fields to the form based on the configuration."""
self.dynamic_fields[collection_name] = []
@@ -17,45 +48,77 @@ class DynamicFormBase(FlaskForm):
current_app.logger.debug(f"{field_name}: {field_def}")
# Prefix the field name with the collection name
full_field_name = f"{collection_name}_{field_name}"
label = field_def.get('name')
field_type = field_def.get('type')
description = field_def.get('description', '')
required = field_def.get('required', False)
default = field_def.get('default')
# Determine validators
field_validators = [validators.InputRequired()] if required else [validators.Optional()]
# Determine standard validators
field_validators = self._create_field_validators(field_def)
# Map the field type to WTForms field classes
field_class = {
'int': IntegerField,
'float': FloatField,
'boolean': BooleanField,
'string': StringField,
}.get(field_type, StringField)
# Handle special case for tagging_fields
if field_type == 'tagging_fields':
field_class = TextAreaField
field_validators.append(validate_tagging_fields)
extra_classes = 'json-editor'
field_kwargs = {}
elif field_type == 'enum':
field_class = SelectField
allowed_values = field_def.get('allowed_values', [])
choices = [(str(val), str(val)) for val in allowed_values]
extra_classes = ''
field_kwargs = {'choices': choices}
else:
extra_classes = ''
field_class = {
'integer': IntegerField,
'float': FloatField,
'boolean': BooleanField,
'string': StringField,
'date': DateField,
}.get(field_type, StringField)
field_kwargs = {}
# Create the field instance
unbound_field = field_class(
label=description,
validators=field_validators,
default=default
)
# Prepare field data
field_data = None
if initial_data and field_name in initial_data:
field_data = initial_data[field_name]
if field_type == 'tagging_fields' and isinstance(field_data, dict):
try:
field_data = json.dumps(field_data, indent=2)
except (TypeError, ValueError) as e:
current_app.logger.error(f"Error converting initial data to JSON: {e}")
field_data = "{}"
elif field_def.get('default') is not None:
field_data = field_def.get('default')
# Create render_kw with classes and any other HTML attributes
render_kw = {'class': extra_classes} if extra_classes else {}
if description:
render_kw['title'] = description # For tooltip
render_kw['data-bs-toggle'] = 'tooltip'
render_kw['data-bs-placement'] = 'right'
# Create the field
field_kwargs.update({
'label': label,
'description': description,
'validators': field_validators,
'default': field_data,
'render_kw': render_kw
})
unbound_field = field_class(**field_kwargs)
# Bind the field to the form
bound_field = unbound_field.bind(form=self, name=full_field_name)
# Process the field with formdata
if self.formdata and full_field_name in self.formdata:
# If formdata is available and contains the field
bound_field.process(self.formdata)
elif initial_data and field_name in initial_data:
# Use initial data if provided
bound_field.process(formdata=None, data=initial_data[field_name])
else:
# Use default value
bound_field.process(formdata=None, data=default)
# Set collection name attribute for identification
# bound_field.collection_name = collection_name
bound_field.process(formdata=None, data=field_data) # Use prepared field_data
# Add the field to the form
setattr(self, full_field_name, bound_field)
@@ -88,5 +151,60 @@ class DynamicFormBase(FlaskForm):
for full_field_name in self.dynamic_fields[collection_name]:
original_field_name = full_field_name[prefix_length:]
field = getattr(self, full_field_name)
data[original_field_name] = field.data
# Parse JSON for tagging_fields type
if isinstance(field, TextAreaField) and field.data:
try:
data[original_field_name] = json.loads(field.data)
except json.JSONDecodeError:
# Validation should catch this, but just in case
data[original_field_name] = field.data
else:
data[original_field_name] = field.data
return data
def validate_tagging_fields(form, field):
"""Validate the tagging fields structure"""
if not field.data:
return
try:
# Parse JSON data
fields_data = json.loads(field.data)
# Validate it's a dictionary
if not isinstance(fields_data, dict):
raise ValidationError("Tagging fields must be a dictionary")
# Validate each field definition
for field_name, field_def in fields_data.items():
if not isinstance(field_def, dict):
raise ValidationError(f"Field definition for {field_name} must be a dictionary")
# Check required properties
if 'type' not in field_def:
raise ValidationError(f"Field {field_name} missing required 'type' property")
# Validate type
if field_def['type'] not in ['string', 'integer', 'float', 'date', 'enum']:
raise ValidationError(f"Field {field_name} has invalid type: {field_def['type']}")
# Validate enum fields have allowed_values
if field_def['type'] == 'enum':
if 'allowed_values' not in field_def:
raise ValidationError(f"Enum field {field_name} missing required 'allowed_values' list")
if not isinstance(field_def['allowed_values'], list):
raise ValidationError(f"Field {field_name} allowed_values must be a list")
# Validate numeric fields
if field_def['type'] in ['integer', 'float']:
if 'min_value' in field_def and 'max_value' in field_def:
min_val = float(field_def['min_value'])
max_val = float(field_def['max_value'])
if min_val >= max_val:
raise ValidationError(f"Field {field_name} min_value must be less than max_value")
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except (TypeError, ValueError) as e:
raise ValidationError(f"Invalid field definition: {str(e)}")