pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Add DOSSIER Catalog management possibilities to eveai_app.

Browse Source
This commit is contained in:
Josako
2025-03-12 11:25:48 +01:00
parent 6465e4f358
commit 56a00c2894
8 changed files with 265 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
eveai_app/views/document_forms.py
View File

@@ -8,6 +8,7 @@ import json
from wtforms_sqlalchemy.fields import QuerySelectField
from common.extensions import cache_manager
from common.models.document import Catalog
from config.type_defs.catalog_types import CATALOG_TYPES
@@ -150,8 +151,9 @@ class RetrieverForm(FlaskForm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
types_dict = cache_manager.retrievers_types_cache.get_types()
# Dynamically populate the 'type' field using the constructor
self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
self.type.choices = [(key, value['name']) for key, value in types_dict.items()]
class EditRetrieverForm(DynamicFormBase):

5
eveai_app/views/document_views.py
View File

@@ -12,7 +12,7 @@ from requests.exceptions import SSLError
import json
from common.models.document import Document, DocumentVersion, Catalog, Retriever, Processor
from common.extensions import db
from common.extensions import db, cache_manager
from common.models.interaction import Specialist, SpecialistRetriever
from common.utils.document_utils import create_document_stack, start_embedding_task, process_url, \
edit_document, \
@@ -303,7 +303,8 @@ def edit_retriever(retriever_id):
# Create form instance with the retriever
form = EditRetrieverForm(request.form, obj=retriever)
configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"]
retriever_config = cache_manager.retrievers_config_cache.get_config(retriever.type, retriever.type_version)
configuration_config = retriever_config.get("configuration")
form.add_dynamic_fields("configuration", configuration_config, retriever.configuration)
if form.validate_on_submit():

175
eveai_app/views/dynamic_form_base.py
View File

@@ -16,17 +16,27 @@ class TaggingFieldsField(TextAreaField):
}
super().__init__(*args, **kwargs)
# def _value(self):
# if self.data:
# return json.dumps(self.data)
# return ''
#
# def process_formdata(self, valuelist):
# if valuelist and valuelist[0]:
# try:
# self.data = json.loads(valuelist[0])
# except json.JSONDecodeError as e:
# raise ValueError('Not valid JSON content')
class TaggingFieldsFilterField(TextAreaField):
"""Field for tagging fields filter conditions"""
def __init__(self, *args, **kwargs):
kwargs['render_kw'] = {
'class': 'json-editor',
'data-handle-enter': 'true'
}
super().__init__(*args, **kwargs)
class DynamicArgumentsField(TextAreaField):
"""Field for dynamic arguments configuration"""
def __init__(self, *args, **kwargs):
kwargs['render_kw'] = {
'class': 'json-editor',
'data-handle-enter': 'true'
}
super().__init__(*args, **kwargs)
class ChunkingPatternsField(TextAreaField):
@@ -37,15 +47,6 @@ class ChunkingPatternsField(TextAreaField):
}
super().__init__(*args, **kwargs)
# def _value(self):
# if self.data:
# return '\n'.join(self.data)
# return ''
#
# def process_formdata(self, valuelist):
# if valuelist and valuelist[0]:
# self.data = [line.strip() for line in valuelist[0].split('\n') if line.strip()]
class DynamicFormBase(FlaskForm):
def __init__(self, formdata=None, *args, **kwargs):
@@ -80,6 +81,10 @@ class DynamicFormBase(FlaskForm):
)
elif field_type == 'tagging_fields':
validators_list.append(self._validate_tagging_fields)
elif field_type == 'tagging_fields_filter':
validators_list.append(self._validate_tagging_fields_filter)
elif field_type == 'dynamic_arguments':
validators_list.append(self._validate_dynamic_arguments)
return validators_list
@@ -103,6 +108,116 @@ class DynamicFormBase(FlaskForm):
except Exception as e:
raise ValidationError(f"Invalid field definition: {str(e)}")
def _validate_tagging_fields_filter(self, form, field):
"""Validate the tagging fields filter structure"""
if not field.data:
return
try:
# Parse JSON data
filter_data = json.loads(field.data)
# Basic validation of filter structure
self._validate_filter_condition(filter_data)
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except ValidationError as e:
# Re-raise ValidationError from _validate_filter_condition
raise e
except Exception as e:
raise ValidationError(f"Invalid filter definition: {str(e)}")
def _validate_filter_condition(self, condition):
"""Recursively validate a filter condition structure"""
# Check if this is a logical condition (AND/OR/NOT)
if 'logical' in condition:
if condition['logical'] not in ['and', 'or', 'not']:
raise ValidationError(f"Invalid logical operator: {condition['logical']}")
if 'conditions' not in condition:
raise ValidationError("Missing 'conditions' array for logical operator")
if not isinstance(condition['conditions'], list):
raise ValidationError("'conditions' must be an array")
# Special case for NOT which should have exactly one condition
if condition['logical'] == 'not' and len(condition['conditions']) != 1:
raise ValidationError("'not' operator must have exactly one condition")
# Validate each sub-condition
for sub_condition in condition['conditions']:
self._validate_filter_condition(sub_condition)
# Check if this is a field condition
elif 'field' in condition:
if 'operator' not in condition:
raise ValidationError(f"Missing 'operator' for field condition on {condition['field']}")
if 'value' not in condition:
raise ValidationError(f"Missing 'value' for field condition on {condition['field']}")
# Validate operator types
# This is a simplified check - in a real implementation, you would validate
# against the actual field type from the catalog definition
valid_operators = {
'string': ['eq', 'neq', 'contains', 'not_contains', 'starts_with',
'ends_with', 'in', 'not_in', 'regex', 'not_regex'],
'numeric': ['eq', 'neq', 'gt', 'gte', 'lt', 'lte', 'between',
'not_between', 'in', 'not_in'],
'enum': ['eq', 'neq', 'in', 'not_in']
}
# For now, we just check that the operator is one of the known types
all_operators = set().union(*valid_operators.values())
if condition['operator'] not in all_operators:
raise ValidationError(f"Unknown operator '{condition['operator']}' for field {condition['field']}")
# Validate variable references if present
if isinstance(condition['value'], str) and condition['value'].startswith('$'):
# This is a variable reference - no further validation needed at form time
pass
# Additional validation based on operator type could be added here
else:
raise ValidationError("Filter condition must have either 'logical' or 'field' property")
def _validate_dynamic_arguments(self, form, field):
"""Validate the dynamic arguments structure"""
if not field.data:
return
try:
# Parse JSON data
args_data = json.loads(field.data)
# Validate basic structure (should be an object with argument definitions)
if not isinstance(args_data, dict):
raise ValidationError("Dynamic arguments must be an object with argument definitions")
# Validate each argument definition
for arg_name, arg_def in args_data.items():
if not isinstance(arg_def, dict):
raise ValidationError(f"Argument definition for '{arg_name}' must be an object")
# Check required properties
if 'type' not in arg_def:
raise ValidationError(f"Argument '{arg_name}' missing required 'type' property")
# Validate type
if arg_def['type'] not in ['string', 'integer', 'float', 'boolean', 'date', 'enum', 'object', 'array']:
raise ValidationError(f"Argument '{arg_name}' has invalid type: {arg_def['type']}")
# Validate enum fields have allowed_values
if arg_def['type'] == 'enum' and 'allowed_values' not in arg_def:
raise ValidationError(f"Enum argument '{arg_name}' missing required 'allowed_values' list")
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except Exception as e:
raise ValidationError(f"Invalid argument definition: {str(e)}")
def add_dynamic_fields(self, collection_name, config, initial_data=None):
"""Add dynamic fields to the form based on the configuration."""
self.dynamic_fields[collection_name] = []
@@ -117,11 +232,19 @@ class DynamicFormBase(FlaskForm):
# Determine standard validators
field_validators = self._create_field_validators(field_def)
# Handle special case for tagging_fields
# Handle special case for field types
if field_type == 'tagging_fields':
field_class = TaggingFieldsField
extra_classes = 'json-editor'
field_kwargs = {}
elif field_type == 'tagging_fields_filter':
field_class = TaggingFieldsFilterField
extra_classes = 'json-editor'
field_kwargs = {}
elif field_type == 'dynamic_arguments':
field_class = DynamicArgumentsField
extra_classes = 'json-editor'
field_kwargs = {}
elif field_type == 'enum':
field_class = SelectField
allowed_values = field_def.get('allowed_values', [])
@@ -148,7 +271,8 @@ class DynamicFormBase(FlaskForm):
field_data = None
if initial_data and field_name in initial_data:
field_data = initial_data[field_name]
if field_type == 'tagging_fields' and isinstance(field_data, dict):
if field_type in ['tagging_fields', 'tagging_fields_filter', 'dynamic_arguments'] and isinstance(
field_data, dict):
try:
field_data = json.dumps(field_data, indent=2)
except (TypeError, ValueError) as e:
@@ -221,8 +345,8 @@ class DynamicFormBase(FlaskForm):
for full_field_name in self.dynamic_fields[collection_name]:
original_field_name = full_field_name[prefix_length:]
field = getattr(self, full_field_name)
# Parse JSON for tagging_fields type
if isinstance(field, TaggingFieldsField) and field.data:
# Parse JSON for special field types
if isinstance(field, (TaggingFieldsField, TaggingFieldsFilterField, DynamicArgumentsField)) and field.data:
try:
data[original_field_name] = json.loads(field.data)
except json.JSONDecodeError:
@@ -282,5 +406,4 @@ def validate_tagging_fields(form, field):
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except (TypeError, ValueError) as e:
raise ValidationError(f"Invalid field definition: {str(e)}")
raise ValidationError(f"Invalid field definition: {str(e)}")