- Refining & Enhancing dynamic fields

- Creating a specialized Form class for handling dynamic fields - Refinement of HTML-macros to handle dynamic fields - Introduction of dynamic fields for Catalogs
2024-10-29 09:17:44 +01:00
parent aa358df28e
commit 43547287b1
13 changed files with 605 additions and 129 deletions
--- a/eveai_app/templates/document/edit_catalog.html
+++ b/eveai_app/templates/document/edit_catalog.html
@@ -11,12 +11,22 @@ When you change chunking of embedding information, you'll need to manually refre
 {% block content %}
    <form method="post">
        {{ form.hidden_tag() }}
-        {%  set disabled_fields = [] %}
-        {%  set exclude_fields = [] %}
-        {% for field in form %}
+        {% set disabled_fields = ['type'] %}
+        {% set exclude_fields = [] %}
+        <!-- Render Static Fields -->
+        {% for field in form.get_static_fields() %}
            {{ render_field(field, disabled_fields, exclude_fields) }}
        {% endfor %}
-        <button type="submit" class="btn btn-primary">Save Catalog</button>
+        <!-- Render Dynamic Fields -->
+        {% for collection_name, fields in form.get_dynamic_fields().items() %}
+            {% if fields|length > 0 %}
+                <h4 class="mt-4">{{ collection_name }}</h4>
+            {% endif %}
+            {% for field in fields %}
+                {{ render_field(field, disabled_fields, exclude_fields) }}
+            {% endfor %}
+        {% endfor %}
+        <button type="submit" class="btn btn-primary">Save Retriever</button>
    </form>
 {% endblock %}

--- a/eveai_app/templates/document/edit_retriever.html
+++ b/eveai_app/templates/document/edit_retriever.html
@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-{% from "macros.html" import render_field2, render_dynamic_fields %}
+{% from "macros.html" import render_field %}

 {% block title %}Edit Retriever{% endblock %}

@@ -13,13 +13,15 @@
        {% set exclude_fields = [] %}
        <!-- Render Static Fields -->
        {% for field in form.get_static_fields() %}
-            {{ render_field2(field, disabled_fields, exclude_fields) }}
+            {{ render_field(field, disabled_fields, exclude_fields) }}
        {% endfor %}
        <!-- Render Dynamic Fields -->
        {% for collection_name, fields in form.get_dynamic_fields().items() %}
-            <h4 class="mt-4">{{ collection_name }}</h4>
+            {% if fields|length > 0 %}
+                <h4 class="mt-4">{{ collection_name }}</h4>
+            {% endif %}
            {% for field in fields %}
-                {{ render_field2(field, disabled_fields, exclude_fields) }}
+                {{ render_field(field, disabled_fields, exclude_fields) }}
            {% endfor %}
        {% endfor %}
        <button type="submit" class="btn btn-primary">Save Retriever</button>
--- a/eveai_app/templates/macros.html
+++ b/eveai_app/templates/macros.html
@@ -1,29 +1,29 @@
-{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %}
-    {% set disabled = field.name in disabled_fields %}
-    {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %}
-    {% if field.name not in exclude_fields %}
-        {% if field.type == 'BooleanField' %}
-            <div class="form-check">
-                {{ field(class="form-check-input " + class, type="checkbox", id="flexSwitchCheckDefault") }}
-                {{ field.label(class="form-check-label", for="flexSwitchCheckDefault", disabled=disabled) }}
-            </div>
-        {% else %}
-            <div class="form-group">
-                {{ field.label(class="form-label") }}
-                {{ field(class="form-control " + class, disabled=disabled) }}
-                {% if field.errors %}
-                    <div class="invalid-feedback">
-                        {% for error in field.errors %}
-                            {{ error }}
-                        {% endfor %}
-                    </div>
-                {% endif %}
-            </div>
-        {% endif %}
-    {% endif %}
-{% endmacro %}
+<!--{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %}-->
+<!--    {% set disabled = field.name in disabled_fields %}-->
+<!--    {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %}-->
+<!--    {% if field.name not in exclude_fields %}-->
+<!--        {% if field.type == 'BooleanField' %}-->
+<!--            <div class="form-check">-->
+<!--                {{ field(class="form-check-input " + class, type="checkbox", id="flexSwitchCheckDefault") }}-->
+<!--                {{ field.label(class="form-check-label", for="flexSwitchCheckDefault", disabled=disabled) }}-->
+<!--            </div>-->
+<!--        {% else %}-->
+<!--            <div class="form-group">-->
+<!--                {{ field.label(class="form-label") }}-->
+<!--                {{ field(class="form-control " + class, disabled=disabled) }}-->
+<!--                {% if field.errors %}-->
+<!--                    <div class="invalid-feedback">-->
+<!--                        {% for error in field.errors %}-->
+<!--                            {{ error }}-->
+<!--                        {% endfor %}-->
+<!--                    </div>-->
+<!--                {% endif %}-->
+<!--            </div>-->
+<!--        {% endif %}-->
+<!--    {% endif %}-->
+<!--{% endmacro %}-->

-{% macro render_field2(field, disabled_fields=[], exclude_fields=[], class='') %}
+{% macro render_field_old(field, disabled_fields=[], exclude_fields=[], class='') %}
    <!-- Debug info -->
    <!-- Field name: {{ field.name }}, Field type: {{ field.__class__.__name__ }} -->

@@ -34,7 +34,14 @@
            <div class="form-group">
                <div class="form-check form-switch">
                    {{ field(class="form-check-input " + class, disabled=disabled) }}
-                    {{ field.label(class="form-check-label") }}
+                    {% if field.description %}
+                        {{ field.label(class="form-check-label",
+                                     **{'data-bs-toggle': 'tooltip',
+                                        'data-bs-placement': 'right',
+                                        'title': field.description}) }}
+                    {% else %}
+                        {{ field.label(class="form-check-label") }}
+                    {% endif %}
                </div>
                {% if field.errors %}
                    <div class="invalid-feedback d-block">
@@ -46,8 +53,82 @@
            </div>
        {% else %}
            <div class="form-group">
-                {{ field.label(class="form-label") }}
-                {{ field(class="form-control " + class, disabled=disabled) }}
+                {% if field.description %}
+                    {{ field.label(class="form-label",
+                                 **{'data-bs-toggle': 'tooltip',
+                                    'data-bs-placement': 'right',
+                                    'title': field.description}) }}
+                {% else %}
+                    {{ field.label(class="form-label") }}
+                {% endif %}
+
+                {% if field.type == 'TextAreaField' and 'json-editor' in class %}
+                    <div id="{{ field.id }}-editor" class="json-editor-container"></div>
+                    {{ field(class="form-control d-none " + class, disabled=disabled) }}
+                {% else %}
+                    {{ field(class="form-control " + class, disabled=disabled) }}
+                {% endif %}
+
+                {% if field.errors %}
+                    <div class="invalid-feedback d-block">
+                        {% for error in field.errors %}
+                            {{ error }}
+                        {% endfor %}
+                    </div>
+                {% endif %}
+            </div>
+        {% endif %}
+    {% endif %}
+{% endmacro %}
+
+{% macro render_field(field, disabled_fields=[], exclude_fields=[], class='') %}
+    <!-- Debug info -->
+    <!-- Field name: {{ field.name }}, Field type: {{ field.__class__.__name__ }} -->
+
+    {% set disabled = field.name in disabled_fields %}
+    {% set exclude_fields = exclude_fields + ['csrf_token', 'submit'] %}
+    {% if field.name not in exclude_fields %}
+        {% if field.type == 'BooleanField' %}
+            <div class="form-group">
+                <div class="form-check form-switch">
+                    {{ field(class="form-check-input " + class, disabled=disabled) }}
+                    {% if field.description %}
+                        {{ field.label(class="form-check-label",
+                                     **{'data-bs-toggle': 'tooltip',
+                                        'data-bs-placement': 'right',
+                                        'title': field.description}) }}
+                    {% else %}
+                        {{ field.label(class="form-check-label") }}
+                    {% endif %}
+                </div>
+                {% if field.errors %}
+                    <div class="invalid-feedback d-block">
+                        {% for error in field.errors %}
+                            {{ error }}
+                        {% endfor %}
+                    </div>
+                {% endif %}
+            </div>
+        {% else %}
+            <div class="form-group">
+                {% if field.description %}
+                    {{ field.label(class="form-label",
+                                 **{'data-bs-toggle': 'tooltip',
+                                    'data-bs-placement': 'right',
+                                    'title': field.description}) }}
+                {% else %}
+                    {{ field.label(class="form-label") }}
+                {% endif %}
+
+                {% if field.type == 'TextAreaField' and 'json-editor' in class %}
+                    <div id="{{ field.id }}-editor" class="json-editor-container"></div>
+                    {{ field(class="form-control d-none " + class, disabled=disabled) }}
+                {% elif field.type == 'SelectField' %}
+                    {{ field(class="form-control form-select " + class, disabled=disabled) }}
+                {% else %}
+                    {{ field(class="form-control " + class, disabled=disabled) }}
+                {% endif %}
+
                {% if field.errors %}
                    <div class="invalid-feedback d-block">
                        {% for error in field.errors %}
--- a/eveai_app/templates/scripts.html
+++ b/eveai_app/templates/scripts.html
@@ -14,4 +14,58 @@
        <script src="{{url_for('static', filename='assets/js/material-kit-pro.min.js')}}?v=3.0.4 type="text/javascript"></script>
        <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.3/js/bootstrap.bundle.min.js"></script>
        <script src="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.13/js/select2.min.js"></script>
+        <link href="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/10.1.0/jsoneditor.min.css" rel="stylesheet" type="text/css">
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/jsoneditor/10.1.0/jsoneditor.min.js"></script>

+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    // Initialize tooltips
+    var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]'))
+    var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) {
+        return new bootstrap.Tooltip(tooltipTriggerEl)
+    });
+
+    // Initialize JSON editors
+    document.querySelectorAll('.json-editor').forEach(function(textarea) {
+        // Create container for editor
+        var container = document.getElementById(textarea.id + '-editor');
+
+        // Initialize the editor
+        var editor = new JSONEditor(container, {
+            mode: 'code',
+            modes: ['code', 'tree'],
+            onChangeText: function(jsonString) {
+                textarea.value = jsonString;
+            }
+        });
+
+        // Set initial value
+        try {
+            const initialValue = textarea.value ? JSON.parse(textarea.value) : {};
+            editor.set(initialValue);
+        } catch (e) {
+            console.error('Error parsing initial JSON:', e);
+            editor.set({});
+        }
+
+        // Add validation indicator
+        editor.validate().then(function(errors) {
+            if (errors.length) {
+                container.style.border = '2px solid red';
+            } else {
+                container.style.border = '1px solid #ccc';
+            }
+        });
+    });
+});
+</script>
+
+<style>
+.json-editor-container {
+    height: 400px;
+    margin-bottom: 1rem;
+}
+.tooltip {
+    position: fixed;
+}
+</style>
--- a/eveai_app/views/document_forms.py
+++ b/eveai_app/views/document_forms.py
@@ -8,6 +8,7 @@ import json

 from wtforms_sqlalchemy.fields import QuerySelectField

+from common.extensions import db
 from common.models.document import Catalog

 from config.catalog_types import CATALOG_TYPES
@@ -34,14 +35,6 @@ def validate_json(form, field):
 class CatalogForm(FlaskForm):
    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
    description = TextAreaField('Description', validators=[Optional()])
-    # Parent ID (Optional for root-level catalogs)
-    parent = QuerySelectField(
-        'Parent Catalog',
-        query_factory=lambda: Catalog.query.all(),
-        allow_blank=True,
-        get_label='name',
-        validators=[Optional()],
-    )

    # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
    type = SelectField('Catalog Type', validators=[DataRequired()])
@@ -56,8 +49,9 @@ class CatalogForm(FlaskForm):
                            default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
    html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
                                default='p, li')
-    html_included_elements = StringField('HTML Included Elements', validators=[Optional()])
-    html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()])
+    html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
+    html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
+                                         default='header, footer, nav, script')
    html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
    min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
                                  default=2000)
@@ -75,6 +69,37 @@ class CatalogForm(FlaskForm):
        self.type.choices = [(key, value['name']) for key, value in CATALOG_TYPES.items()]


+class EditCatalogForm(DynamicFormBase):
+    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
+    description = TextAreaField('Description', validators=[Optional()])
+
+    # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
+    type = StringField('Catalog Type', validators=[DataRequired()], render_kw={'readonly': True})
+
+    # Metadata fields
+    user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
+    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json],)
+
+    # HTML Embedding Variables
+    html_tags = StringField('HTML Tags', validators=[DataRequired()],
+                            default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
+    html_end_tags = StringField('HTML End Tags', validators=[DataRequired()],
+                                default='p, li')
+    html_included_elements = StringField('HTML Included Elements', validators=[Optional()], default='article, main')
+    html_excluded_elements = StringField('HTML Excluded Elements', validators=[Optional()],
+                                         default='header, footer, nav, script')
+    html_excluded_classes = StringField('HTML Excluded Classes', validators=[Optional()])
+    min_chunk_size = IntegerField('Minimum Chunk Size (2000)', validators=[NumberRange(min=0), Optional()],
+                                  default=2000)
+    max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
+                                  default=3000)
+    # Chat Variables
+    chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)])
+    chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)])
+    # Tuning variables
+    embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
+
+
 class RetrieverForm(FlaskForm):
    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
    description = TextAreaField('Description', validators=[Optional()])
--- a/eveai_app/views/document_views.py
+++ b/eveai_app/views/document_views.py
@@ -22,14 +22,14 @@ from common.utils.document_utils import validate_file_type, create_document_stac
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
    EveAIDoubleURLException
 from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, AddURLsForm, \
-    CatalogForm, RetrieverForm, EditRetrieverForm
+    CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm
 from common.utils.middleware import mw_before_request
 from common.utils.celery_utils import current_celery
 from common.utils.nginx_utils import prefixed_url_for
 from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro, form_to_dict
 from .document_list_view import DocumentListView
 from .document_version_list_view import DocumentVersionListView
-
+from config.catalog_types import CATALOG_TYPES
 from config.retriever_types import RETRIEVER_TYPES

 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
@@ -67,7 +67,6 @@ def catalog():
        tenant_id = session.get('tenant').get('id')
        new_catalog = Catalog()
        form.populate_obj(new_catalog)
-        new_catalog.parent_id = form.parent.data.get('id')
        # Handle Embedding Variables
        new_catalog.html_tags = [tag.strip() for tag in form.html_tags.data.split(',')] if form.html_tags.data else []
        new_catalog.html_end_tags = [tag.strip() for tag in form.html_end_tags.data.split(',')] \
@@ -135,9 +134,12 @@ def handle_catalog_selection():
@roles_accepted('Super User', 'Tenant Admin')
 def edit_catalog(catalog_id):
    catalog = Catalog.query.get_or_404(catalog_id)
-    form = CatalogForm(obj=catalog)
    tenant_id = session.get('tenant').get('id')

+    form = EditCatalogForm(request.form, obj=catalog)
+    configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
+    form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
+
    # Convert arrays to comma-separated strings for display
    if request.method == 'GET':
        form.html_tags.data = ', '.join(catalog.html_tags or '')
@@ -158,6 +160,8 @@ def edit_catalog(catalog_id):
            if form.html_excluded_elements.data else []
        catalog.html_excluded_classes = [cls.strip() for cls in form.html_excluded_classes.data.split(',')] \
            if form.html_excluded_classes.data else []
+
+        catalog.configuration = form.get_dynamic_data('configuration')
        update_logging_information(catalog, dt.now(tz.utc))
        try:
            db.session.add(catalog)
@@ -210,8 +214,6 @@ def retriever():
@roles_accepted('Super User', 'Tenant Admin')
 def edit_retriever(retriever_id):
    """Edit an existing retriever configuration."""
-    current_app.logger.debug(f"Editing Retriever {retriever_id}")
-
    # Get the retriever or return 404
    retriever = Retriever.query.get_or_404(retriever_id)

@@ -225,7 +227,6 @@ def edit_retriever(retriever_id):
    form = EditRetrieverForm(request.form, obj=retriever)

    configuration_config = RETRIEVER_TYPES[retriever.type]["configuration"]
-    current_app.logger.debug(f"Configuration {configuration_config}")
    form.add_dynamic_fields("configuration", configuration_config, retriever.configuration)

    if form.validate_on_submit():
@@ -663,49 +664,3 @@ def fetch_html(url):

    response.raise_for_status()  # Will raise an exception for bad requests
    return response.content
-
-
-# def prepare_document_data(docs):
-#     rows = []
-#     for doc in docs:
-#         doc_row = [{'value': doc.name, 'class': '', 'type': 'text'},
-#                    {'value': doc.created_at.strftime("%Y-%m-%d %H:%M:%S"), 'class': '', 'type': 'text'}]
-#         # Document basic details
-#         if doc.valid_from:
-#             doc_row.append({'value': doc.valid_from.strftime("%Y-%m-%d"), 'class': '', 'type': 'text'})
-#         else:
-#             doc_row.append({'value': '', 'class': '', 'type': 'text'})
-#
-#         # Nested languages and versions
-#         languages_rows = []
-#         for lang in doc.languages:
-#             lang_row = [{'value': lang.language, 'class': '', 'type': 'text'}]
-#
-#             # Latest version details if available (should be available ;-) )
-#             if lang.latest_version:
-#                 lang_row.append({'value': lang.latest_version.created_at.strftime("%Y-%m-%d %H:%M:%S"),
-#                                  'class': '', 'type': 'text'})
-#                 if lang.latest_version.url:
-#                     lang_row.append({'value': lang.latest_version.url,
-#                                      'class': '', 'type': 'link', 'href': lang.latest_version.url})
-#                 else:
-#                     lang_row.append({'value': '', 'class': '', 'type': 'text'})
-#
-#                 if lang.latest_version.object_name:
-#                     lang_row.append({'value': lang.latest_version.object_name, 'class': '', 'type': 'text'})
-#                 else:
-#                     lang_row.append({'value': '', 'class': '', 'type': 'text'})
-#
-#                 if lang.latest_version.file_type:
-#                     lang_row.append({'value': lang.latest_version.file_type, 'class': '', 'type': 'text'})
-#                 else:
-#                     lang_row.append({'value': '', 'class': '', 'type': 'text'})
-#                     # Include other details as necessary
-#
-#             languages_rows.append(lang_row)
-#
-#         doc_row.append({'is_group': True, 'colspan': '5',
-#                         'headers': ['Language', 'Latest Version', 'URL', 'File Name', 'Type'],
-#                         'sub_rows': languages_rows})
-#         rows.append(doc_row)
-#     return rows
--- a/eveai_app/views/dynamic_form_base.py
+++ b/eveai_app/views/dynamic_form_base.py
@@ -1,6 +1,11 @@
 from flask_wtf import FlaskForm
-from wtforms import IntegerField, FloatField, BooleanField, StringField, validators
+from wtforms import IntegerField, FloatField, BooleanField, StringField, TextAreaField, validators, ValidationError
 from flask import current_app
+import json
+
+from wtforms.fields.choices import SelectField
+from wtforms.fields.datetime import DateField
+

 class DynamicFormBase(FlaskForm):
    def __init__(self, formdata=None, *args, **kwargs):
@@ -10,6 +15,32 @@ class DynamicFormBase(FlaskForm):
        # Store formdata for later use
        self.formdata = formdata

+    def _create_field_validators(self, field_def):
+        """Create validators based on field definition"""
+        validators_list = []
+
+        # Required validator
+        if field_def.get('required', False):
+            validators_list.append(validators.InputRequired())
+        else:
+            validators_list.append(validators.Optional())
+
+        # Type-specific validators
+        field_type = field_def.get('type')
+        if field_type in ['integer', 'float']:
+            min_value = field_def.get('min_value')
+            max_value = field_def.get('max_value')
+            if min_value is not None or max_value is not None:
+                validators_list.append(
+                    validators.NumberRange(
+                        min=min_value if min_value is not None else -float('inf'),
+                        max=max_value if max_value is not None else float('inf'),
+                        message=f"Value must be between {min_value or '-∞'} and {max_value or '∞'}"
+                    )
+                )
+
+        return validators_list
+
    def add_dynamic_fields(self, collection_name, config, initial_data=None):
        """Add dynamic fields to the form based on the configuration."""
        self.dynamic_fields[collection_name] = []
@@ -17,45 +48,77 @@ class DynamicFormBase(FlaskForm):
            current_app.logger.debug(f"{field_name}: {field_def}")
            # Prefix the field name with the collection name
            full_field_name = f"{collection_name}_{field_name}"
+            label = field_def.get('name')
            field_type = field_def.get('type')
            description = field_def.get('description', '')
            required = field_def.get('required', False)
            default = field_def.get('default')

-            # Determine validators
-            field_validators = [validators.InputRequired()] if required else [validators.Optional()]
+            # Determine standard validators
+            field_validators = self._create_field_validators(field_def)

-            # Map the field type to WTForms field classes
-            field_class = {
-                'int': IntegerField,
-                'float': FloatField,
-                'boolean': BooleanField,
-                'string': StringField,
-            }.get(field_type, StringField)
+            # Handle special case for tagging_fields
+            if field_type == 'tagging_fields':
+                field_class = TextAreaField
+                field_validators.append(validate_tagging_fields)
+                extra_classes = 'json-editor'
+                field_kwargs = {}
+            elif field_type == 'enum':
+                field_class = SelectField
+                allowed_values = field_def.get('allowed_values', [])
+                choices = [(str(val), str(val)) for val in allowed_values]
+                extra_classes = ''
+                field_kwargs = {'choices': choices}
+            else:
+                extra_classes = ''
+                field_class = {
+                    'integer': IntegerField,
+                    'float': FloatField,
+                    'boolean': BooleanField,
+                    'string': StringField,
+                    'date': DateField,
+                }.get(field_type, StringField)
+                field_kwargs = {}

-            # Create the field instance
-            unbound_field = field_class(
-                label=description,
-                validators=field_validators,
-                default=default
-            )
+            # Prepare field data
+            field_data = None
+            if initial_data and field_name in initial_data:
+                field_data = initial_data[field_name]
+                if field_type == 'tagging_fields' and isinstance(field_data, dict):
+                    try:
+                        field_data = json.dumps(field_data, indent=2)
+                    except (TypeError, ValueError) as e:
+                        current_app.logger.error(f"Error converting initial data to JSON: {e}")
+                        field_data = "{}"
+            elif field_def.get('default') is not None:
+                field_data = field_def.get('default')
+
+            # Create render_kw with classes and any other HTML attributes
+            render_kw = {'class': extra_classes} if extra_classes else {}
+            if description:
+                render_kw['title'] = description  # For tooltip
+                render_kw['data-bs-toggle'] = 'tooltip'
+                render_kw['data-bs-placement'] = 'right'
+
+            # Create the field
+            field_kwargs.update({
+                'label': label,
+                'description': description,
+                'validators': field_validators,
+                'default': field_data,
+                'render_kw': render_kw
+            })
+
+            unbound_field = field_class(**field_kwargs)

            # Bind the field to the form
            bound_field = unbound_field.bind(form=self, name=full_field_name)

            # Process the field with formdata
            if self.formdata and full_field_name in self.formdata:
-                # If formdata is available and contains the field
                bound_field.process(self.formdata)
-            elif initial_data and field_name in initial_data:
-                # Use initial data if provided
-                bound_field.process(formdata=None, data=initial_data[field_name])
            else:
-                # Use default value
-                bound_field.process(formdata=None, data=default)
-
-            # Set collection name attribute for identification
-            # bound_field.collection_name = collection_name
+                bound_field.process(formdata=None, data=field_data)  # Use prepared field_data

            # Add the field to the form
            setattr(self, full_field_name, bound_field)
@@ -88,5 +151,60 @@ class DynamicFormBase(FlaskForm):
        for full_field_name in self.dynamic_fields[collection_name]:
            original_field_name = full_field_name[prefix_length:]
            field = getattr(self, full_field_name)
-            data[original_field_name] = field.data
+            # Parse JSON for tagging_fields type
+            if isinstance(field, TextAreaField) and field.data:
+                try:
+                    data[original_field_name] = json.loads(field.data)
+                except json.JSONDecodeError:
+                    # Validation should catch this, but just in case
+                    data[original_field_name] = field.data
+            else:
+                data[original_field_name] = field.data
        return data
+
+
+def validate_tagging_fields(form, field):
+    """Validate the tagging fields structure"""
+    if not field.data:
+        return
+
+    try:
+        # Parse JSON data
+        fields_data = json.loads(field.data)
+
+        # Validate it's a dictionary
+        if not isinstance(fields_data, dict):
+            raise ValidationError("Tagging fields must be a dictionary")
+
+        # Validate each field definition
+        for field_name, field_def in fields_data.items():
+            if not isinstance(field_def, dict):
+                raise ValidationError(f"Field definition for {field_name} must be a dictionary")
+
+            # Check required properties
+            if 'type' not in field_def:
+                raise ValidationError(f"Field {field_name} missing required 'type' property")
+
+            # Validate type
+            if field_def['type'] not in ['string', 'integer', 'float', 'date', 'enum']:
+                raise ValidationError(f"Field {field_name} has invalid type: {field_def['type']}")
+
+            # Validate enum fields have allowed_values
+            if field_def['type'] == 'enum':
+                if 'allowed_values' not in field_def:
+                    raise ValidationError(f"Enum field {field_name} missing required 'allowed_values' list")
+                if not isinstance(field_def['allowed_values'], list):
+                    raise ValidationError(f"Field {field_name} allowed_values must be a list")
+
+            # Validate numeric fields
+            if field_def['type'] in ['integer', 'float']:
+                if 'min_value' in field_def and 'max_value' in field_def:
+                    min_val = float(field_def['min_value'])
+                    max_val = float(field_def['max_value'])
+                    if min_val >= max_val:
+                        raise ValidationError(f"Field {field_name} min_value must be less than max_value")
+
+    except json.JSONDecodeError:
+        raise ValidationError("Invalid JSON format")
+    except (TypeError, ValueError) as e:
+        raise ValidationError(f"Invalid field definition: {str(e)}")