pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Replace old implementation of PROCESSOR_TYPES and CATALOG_TYPES with the new cached approach

Browse Source 
- Add an ordered_list dynamic field type (to be refined)
- Add tabulator javascript library to project
This commit is contained in:
Josako
2025-05-29 16:00:25 +02:00
parent 8a29eb0d8f
commit 25e169dbea
23 changed files with 687 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
eveai_app/views/document_views.py
View File

@@ -20,7 +20,6 @@ from common.utils.document_utils import create_document_stack, start_embedding_t
from common.utils.dynamic_field_utils import create_default_config_from_type_config
from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
EveAIDoubleURLException, EveAIException
from config.type_defs.processor_types import PROCESSOR_TYPES
from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, \
CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
from common.utils.middleware import mw_before_request
@@ -29,7 +28,6 @@ from common.utils.nginx_utils import prefixed_url_for
from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
from .document_list_view import DocumentListView
from .document_version_list_view import DocumentVersionListView
from config.type_defs.catalog_types import CATALOG_TYPES
document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -126,8 +124,8 @@ def edit_catalog(catalog_id):
tenant_id = session.get('tenant').get('id')
form = EditCatalogForm(request.form, obj=catalog)
configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
form.add_dynamic_fields("configuration", full_config, catalog.configuration)
if request.method == 'POST' and form.validate_on_submit():
form.populate_obj(catalog)
@@ -160,8 +158,9 @@ def processor():
new_processor = Processor()
form.populate_obj(new_processor)
new_processor.catalog_id = form.catalog.data.id
processor_config = cache_manager.processors_config_cache.get_config(new_processor.type)
new_processor.configuration = create_default_config_from_type_config(
PROCESSOR_TYPES[new_processor.type]["configuration"])
processor_config["configuration"])
set_logging_information(new_processor, dt.now(tz.utc))
@@ -197,8 +196,8 @@ def edit_processor(processor_id):
# Create form instance with the processor
form = EditProcessorForm(request.form, obj=processor)
configuration_config = PROCESSOR_TYPES[processor.type]["configuration"]
form.add_dynamic_fields("configuration", configuration_config, processor.configuration)
full_config = cache_manager.processors_config_cache.get_config(processor.type)
form.add_dynamic_fields("configuration", full_config, processor.configuration)
if form.validate_on_submit():
# Update basic fields
@@ -390,9 +389,10 @@ def add_document():
catalog = Catalog.query.get_or_404(catalog_id)
if catalog.configuration and len(catalog.configuration) > 0:
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
document_version_configurations = full_config['document_version_configurations']
for config in document_version_configurations:
form.add_dynamic_fields(config, catalog.configuration[config])
form.add_dynamic_fields(config, full_config, catalog.configuration[config])
if form.validate_on_submit():
try:
@@ -403,7 +403,8 @@ def add_document():
filename = secure_filename(file.filename)
extension = filename.rsplit('.', 1)[1].lower()
catalog_properties = {}
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
document_version_configurations = full_config['document_version_configurations']
for config in document_version_configurations:
catalog_properties[config] = form.get_dynamic_data(config)
@@ -445,9 +446,10 @@ def add_url():
catalog = Catalog.query.get_or_404(catalog_id)
if catalog.configuration and len(catalog.configuration) > 0:
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
document_version_configurations = full_config['document_version_configurations']
for config in document_version_configurations:
form.add_dynamic_fields(config, catalog.configuration[config])
form.add_dynamic_fields(config, full_config, catalog.configuration[config])
if form.validate_on_submit():
try:
@@ -459,7 +461,8 @@ def add_url():
file_content, filename, extension = process_url(url, tenant_id)
catalog_properties = {}
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
document_version_configurations = full_config['document_version_configurations']
for config in document_version_configurations:
catalog_properties[config] = form.get_dynamic_data(config)
@@ -582,13 +585,14 @@ def edit_document_version_view(document_version_id):
catalog = Catalog.query.get_or_404(catalog_id)
if catalog.configuration and len(catalog.configuration) > 0:
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
document_version_configurations = full_config['document_version_configurations']
for config in document_version_configurations:
form.add_dynamic_fields(config, catalog.configuration[config], doc_vers.catalog_properties[config])
form.add_dynamic_fields(config, full_config, doc_vers.catalog_properties[config])
if form.validate_on_submit():
catalog_properties = {}
document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
# Use the full_config variable we already defined
for config in document_version_configurations:
catalog_properties[config] = form.get_dynamic_data(config)
@@ -897,4 +901,3 @@ def clean_markdown(markdown):
if markdown.endswith("```"):
markdown = markdown[:-3].strip()
return markdown

179
eveai_app/views/dynamic_form_base.py
View File

@@ -49,6 +49,51 @@ class ChunkingPatternsField(TextAreaField):
super().__init__(*args, **kwargs)
class OrderedListField(TextAreaField):
"""Field for ordered list data that will be rendered as a Tabulator table"""
def __init__(self, *args, **kwargs):
list_type = kwargs.pop('list_type', '')
# Behoud bestaande render_kw attributen als die er zijn
if 'render_kw' in kwargs:
existing_render_kw = kwargs['render_kw']
else:
existing_render_kw = {}
current_app.logger.debug(f"incomming render_kw for ordered list field: {existing_render_kw}")
# Stel nieuwe render_kw samen
new_render_kw = {
'data-list-type': list_type,
'data-handle-enter': 'true'
}
# Voeg klasse toe en behoud bestaande klassen
if 'class' in existing_render_kw:
existing_classes = existing_render_kw['class']
if isinstance(existing_classes, list):
existing_classes += ' ordered-list-field'
new_render_kw['class'] = existing_classes
else:
# String classes samenvoegen
new_render_kw['class'] = f"{existing_classes} ordered-list-field"
else:
new_render_kw['class'] = 'ordered-list-field'
# Voeg alle bestaande attributen toe aan nieuwe render_kw
for key, value in existing_render_kw.items():
if key != 'class': # Klassen hebben we al verwerkt
new_render_kw[key] = value
current_app.logger.debug(f"final render_kw for ordered list field: {new_render_kw}")
# Update kwargs met de nieuwe gecombineerde render_kw
kwargs['render_kw'] = new_render_kw
super().__init__(*args, **kwargs)
class DynamicFormBase(FlaskForm):
def __init__(self, formdata=None, *args, **kwargs):
super(DynamicFormBase, self).__init__(*args, **kwargs)
@@ -89,6 +134,8 @@ class DynamicFormBase(FlaskForm):
validators_list.append(self._validate_tagging_fields_filter)
elif field_type == 'dynamic_arguments':
validators_list.append(self._validate_dynamic_arguments)
elif field_type == 'ordered_list':
validators_list.append(self._validate_ordered_list)
return validators_list
@@ -227,10 +274,52 @@ class DynamicFormBase(FlaskForm):
except Exception as e:
raise ValidationError(f"Invalid argument definition: {str(e)}")
def _validate_ordered_list(self, form, field):
"""Validate the ordered list structure"""
if not field.data:
return
try:
# Parse JSON data
list_data = json.loads(field.data)
# Validate it's a list
if not isinstance(list_data, list):
raise ValidationError("Ordered list must be a list")
# Validate each item in the list is a dictionary
for i, item in enumerate(list_data):
if not isinstance(item, dict):
raise ValidationError(f"Item {i} in ordered list must be an object")
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except Exception as e:
raise ValidationError(f"Invalid ordered list: {str(e)}")
def add_dynamic_fields(self, collection_name, config, initial_data=None):
"""Add dynamic fields to the form based on the configuration."""
"""Add dynamic fields to the form based on the configuration.
Args:
collection_name: The name of the collection of fields to add
config: The full configuration object, which should contain the field definitions
for the collection_name and may also contain list_type definitions
initial_data: Optional initial data for the fields
"""
current_app.logger.debug(f"Adding dynamic fields for collection {collection_name} with config: {config}")
# Store the full configuration for later use in get_list_type_configs_js
if not hasattr(self, '_full_configs'):
self._full_configs = {}
self._full_configs[collection_name] = config
# Get the specific field configuration for this collection
field_config = config.get(collection_name, {})
if not field_config:
# Handle the case where config is already the specific field configuration
return
self.dynamic_fields[collection_name] = []
for field_name, field_def in config.items():
for field_name, field_def in field_config.items():
# Prefix the field name with the collection name
full_field_name = f"{collection_name}_{field_name}"
label = field_def.get('name', field_name)
@@ -264,6 +353,12 @@ class DynamicFormBase(FlaskForm):
field_class = ChunkingPatternsField
extra_classes = ['monospace-text', 'pattern-input']
field_kwargs = {}
elif field_type == 'ordered_list':
current_app.logger.debug(f"Adding ordered list field for {full_field_name}")
field_class = OrderedListField
extra_classes = ''
list_type = field_def.get('list_type', '')
field_kwargs = {'list_type': list_type}
else:
extra_classes = ''
field_class = {
@@ -289,6 +384,12 @@ class DynamicFormBase(FlaskForm):
except (TypeError, ValueError) as e:
current_app.logger.error(f"Error converting initial data to JSON: {e}")
field_data = "{}"
elif field_type == 'ordered_list' and isinstance(field_data, list):
try:
field_data = json.dumps(field_data)
except (TypeError, ValueError) as e:
current_app.logger.error(f"Error converting ordered list data to JSON: {e}")
field_data = "[]"
elif field_type == 'chunking_patterns':
try:
field_data = json_to_patterns(field_data)
@@ -305,6 +406,9 @@ class DynamicFormBase(FlaskForm):
render_kw['data-bs-toggle'] = 'tooltip'
render_kw['data-bs-placement'] = 'right'
current_app.logger.debug(f"render_kw for {full_field_name}: {render_kw}")
# Create the field
field_kwargs.update({
'label': label,
@@ -340,6 +444,73 @@ class DynamicFormBase(FlaskForm):
# Return all fields that are not dynamic
return [field for name, field in self._fields.items() if name not in dynamic_field_names]
def get_list_type_configs_js(self):
"""Generate JavaScript code for list type configurations used by ordered_list fields."""
from common.extensions import cache_manager
list_types = {}
# First check if we have any full configurations stored
if hasattr(self, '_full_configs'):
# Look for list types in the stored full configurations
for config_name, config in self._full_configs.items():
for key, value in config.items():
# Check if this is a list type definition (not a field definition)
if isinstance(value, dict) and all(isinstance(v, dict) for v in value.values()):
# This looks like a list type definition
list_types[key] = value
# Collect all list types used in ordered_list fields
for collection_name, field_names in self.dynamic_fields.items():
for full_field_name in field_names:
field = getattr(self, full_field_name)
if isinstance(field, OrderedListField):
list_type = field.render_kw.get('data-list-type')
if list_type and list_type not in list_types:
# First try to get from current_app.config
list_type_config = current_app.config.get('LIST_TYPES', {}).get(list_type)
if list_type_config:
list_types[list_type] = list_type_config
else:
# Try to find the list type in specialist configurations using the cache
try:
# Get all specialist types
specialist_types = cache_manager.specialists_types_cache.get_types()
# For each specialist type, check if it has the list type we're looking for
for specialist_type in specialist_types:
try:
# Get the latest version for this specialist type
latest_version = cache_manager.specialists_version_tree_cache.get_latest_version(specialist_type)
# Get the configuration for this specialist type and version
specialist_config = cache_manager.specialists_config_cache.get_config(specialist_type, latest_version)
# Check if this specialist has the list type we're looking for
if list_type in specialist_config:
list_types[list_type] = specialist_config[list_type]
break
except Exception as e:
current_app.logger.debug(f"Error checking specialist {specialist_type}: {e}")
continue
except Exception as e:
current_app.logger.error(f"Error retrieving specialist configurations: {e}")
# If no list types found, return empty script
if not list_types:
return ""
# Generate JavaScript code
js_code = "<script>\n"
js_code += "window.listTypeConfigs = window.listTypeConfigs || {};\n"
for list_type, config in list_types.items():
js_code += f"window.listTypeConfigs['{list_type}'] = {json.dumps(config, indent=2)};\n"
js_code += "</script>\n"
return js_code
def get_dynamic_fields(self):
"""Return a dictionary of dynamic fields per collection."""
result = {}
@@ -361,7 +532,7 @@ class DynamicFormBase(FlaskForm):
if field.type == 'BooleanField':
data[original_field_name] = full_field_name in self.raw_formdata
current_app.logger.debug(f"Value for {original_field_name} is {data[original_field_name]}")
elif isinstance(field, (TaggingFieldsField, TaggingFieldsFilterField, DynamicArgumentsField)) and field.data:
elif isinstance(field, (TaggingFieldsField, TaggingFieldsFilterField, DynamicArgumentsField, OrderedListField)) and field.data:
try:
data[original_field_name] = json.loads(field.data)
except json.JSONDecodeError:
@@ -443,4 +614,4 @@ def validate_tagging_fields(form, field):
except json.JSONDecodeError:
raise ValidationError("Invalid JSON format")
except (TypeError, ValueError) as e:
raise ValidationError(f"Invalid field definition: {str(e)}")
raise ValidationError(f"Invalid field definition: {str(e)}")

13
eveai_app/views/interaction_views.py
View File

@@ -204,8 +204,7 @@ def edit_specialist(specialist_id):
form = EditSpecialistForm(request.form, obj=specialist)
specialist_config = cache_manager.specialists_config_cache.get_config(specialist.type, specialist.type_version)
configuration_config = specialist_config.get('configuration')
form.add_dynamic_fields("configuration", configuration_config, specialist.configuration)
form.add_dynamic_fields("configuration", specialist_config, specialist.configuration)
agent_rows = prepare_table_for_macro(specialist.agents,
[('id', ''), ('name', ''), ('type', ''), ('type_version', '')])
@@ -521,8 +520,7 @@ def edit_asset_version(asset_version_id):
form = EditEveAIAssetVersionForm(asset_version)
asset_config = cache_manager.assets_config_cache.get_config(asset_version.asset.type,
asset_version.asset.type_version)
configuration_config = asset_config.get('configuration')
form.add_dynamic_fields("configuration", configuration_config, asset_version.configuration)
form.add_dynamic_fields("configuration", asset_config, asset_version.configuration)
if form.validate_on_submit():
# Update the configuration dynamic fields
@@ -582,9 +580,8 @@ def execute_specialist(specialist_id):
return redirect(prefixed_url_for('interaction_bp.specialists'))
form = ExecuteSpecialistForm(request.form, obj=specialist)
arguments_config = specialist_config.get('arguments', None)
if arguments_config:
form.add_dynamic_fields('arguments', arguments_config)
if 'arguments' in specialist_config:
form.add_dynamic_fields('arguments', specialist_config)
if form.validate_on_submit():
# We're only interested in gathering the dynamic arguments
@@ -674,4 +671,4 @@ def session_interactions(chat_session_id):
This route shows all interactions for a given chat_session_id (int).
"""
chat_session = ChatSession.query.get_or_404(chat_session_id)
return session_interactions_by_session_id(chat_session.session_id)
return session_interactions_by_session_id(chat_session.session_id)