- Replace old implementation of PROCESSOR_TYPES and CATALOG_TYPES with the new cached approach

- Add an ordered_list dynamic field type (to be refined)
- Add tabulator javascript library to project

common/utils/cache/config_cache.py

@@ -7,7 +7,7 @@ from flask import current_app
 
 from common.utils.cache.base import CacheHandler, CacheKey
 from config.type_defs import agent_types, task_types, tool_types, specialist_types, retriever_types, prompt_types, \
-    catalog_types, partner_service_types
+    catalog_types, partner_service_types, processor_types
 
 
 def is_major_minor(version: str) -> bool:
@@ -59,7 +59,7 @@ class BaseConfigCacheHandler(CacheHandler[Dict[str, Any]]):
         """Set the version tree cache dependency."""
         self.version_tree_cache = cache
 
-    def _load_specific_config(self, type_name: str, version_str: str) -> Dict[str, Any]:
+    def _load_specific_config(self, type_name: str, version_str: str = 'latest') -> Dict[str, Any]:
         """
         Load a specific configuration version
         Automatically handles global vs partner-specific configs
@@ -456,6 +456,13 @@ CatalogConfigCacheHandler, CatalogConfigVersionTreeCacheHandler, CatalogConfigTy
         types_module=catalog_types.CATALOG_TYPES
     ))
 
+ProcessorConfigCacheHandler, ProcessorConfigVersionTreeCacheHandler, ProcessorConfigTypesCacheHandler = (
+    create_config_cache_handlers(
+        config_type='processors',
+        config_dir='config/processors',
+        types_module=processor_types.PROCESSOR_TYPES
+    ))
+
 # Add to common/utils/cache/config_cache.py
 PartnerServiceConfigCacheHandler, PartnerServiceConfigVersionTreeCacheHandler, PartnerServiceConfigTypesCacheHandler = (
     create_config_cache_handlers(
@@ -487,6 +494,9 @@ def register_config_cache_handlers(cache_manager) -> None:
     cache_manager.register_handler(CatalogConfigCacheHandler, 'eveai_config')
     cache_manager.register_handler(CatalogConfigTypesCacheHandler, 'eveai_config')
     cache_manager.register_handler(CatalogConfigVersionTreeCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigTypesCacheHandler, 'eveai_config')
+    cache_manager.register_handler(ProcessorConfigVersionTreeCacheHandler, 'eveai_config')
     cache_manager.register_handler(AgentConfigCacheHandler, 'eveai_config')
     cache_manager.register_handler(AgentConfigTypesCacheHandler, 'eveai_config')
     cache_manager.register_handler(AgentConfigVersionTreeCacheHandler, 'eveai_config')
@@ -500,4 +510,6 @@ def register_config_cache_handlers(cache_manager) -> None:
     cache_manager.specialists_config_cache.set_version_tree_cache(cache_manager.specialists_version_tree_cache)
     cache_manager.retrievers_config_cache.set_version_tree_cache(cache_manager.retrievers_version_tree_cache)
     cache_manager.prompts_config_cache.set_version_tree_cache(cache_manager.prompts_version_tree_cache)
+    cache_manager.catalogs_config_cache.set_version_tree_cache(cache_manager.catalogs_version_tree_cache)
+    cache_manager.processors_config_cache.set_version_tree_cache(cache_manager.processors_version_tree_cache)
     cache_manager.partner_services_config_cache.set_version_tree_cache(cache_manager.partner_services_version_tree_cache)

config/catalogs/globals/DOSSIER_CATALOG/1.0.0.yaml

@@ -0,0 +1,21 @@
+version: "1.0.0"
+name: "Dossier Catalog"
+description: "A Catalog with information in Evie's Library in which several Dossiers can be stored"
+configuration:
+  tagging_fields:
+    name: "Tagging Fields"
+    type: "tagging_fields"
+    description: "Define the metadata fields that will be used for tagging documents.
+                Each field must have:
+                - type: one of 'string', 'integer', 'float', 'date', 'enum'
+                - required: boolean indicating if the field is mandatory
+                - description: field description
+                - allowed_values: list of values (for enum type only)
+                - min_value/max_value: range limits (for numeric types only)"
+    required: true
+    default: {}
+document_version_configurations: ["tagging_fields"]
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A Catalog with information in Evie's Library in which several Dossiers can be stored"

config/catalogs/globals/STANDARD_CATALOG/1.0.0.yaml

@@ -0,0 +1,9 @@
+version: "1.0.0"
+name: "Standard Catalog"
+description: "A Catalog with information in Evie's Library, to be considered as a whole"
+configuration: {}
+document_version_configurations: []
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A Catalog with information in Evie's Library, to be considered as a whole"

config/processors/globals/AUDIO_PROCESSOR/1.0.0.yaml

@@ -0,0 +1,9 @@
+version: "1.0.0"
+name: "AUDIO Processor"
+file_types: "mp3, mp4, ogg"
+description: "A Processor for audio files"
+configuration: {}
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A Processor for audio files"

config/processors/globals/DOCX_PROCESSOR/1.0.0.yaml

@@ -0,0 +1,59 @@
+version: "1.0.0"
+name: "DOCX Processor"
+file_types: "docx"
+description: "A processor for DOCX files"
+configuration:
+  chunking_patterns:
+    name: "Chunking Patterns"
+    description: "A list of Patterns used to chunk files into logical pieces"
+    type: "chunking_patterns"
+    required: false
+  chunking_heading_level:
+    name: "Chunking Heading Level"
+    type: "integer"
+    description: "Maximum heading level to consider for chunking (1-6)"
+    required: false
+    default: 2
+  extract_comments:
+    name: "Extract Comments"
+    type: "boolean"
+    description: "Whether to include document comments in the markdown"
+    required: false
+    default: false
+  extract_headers_footers:
+    name: "Extract Headers/Footers"
+    type: "boolean"
+    description: "Whether to include headers and footers in the markdown"
+    required: false
+    default: false
+  preserve_formatting:
+    name: "Preserve Formatting"
+    type: "boolean"
+    description: "Whether to preserve bold, italic, and other text formatting"
+    required: false
+    default: true
+  list_style:
+    name: "List Style"
+    type: "enum"
+    description: "How to format lists in markdown"
+    required: false
+    default: "dash"
+    allowed_values: ["dash", "asterisk", "plus"]
+  image_handling:
+    name: "Image Handling"
+    type: "enum"
+    description: "How to handle embedded images"
+    required: false
+    default: "skip"
+    allowed_values: ["skip", "extract", "placeholder"]
+  table_alignment:
+    name: "Table Alignment"
+    type: "enum"
+    description: "How to align table contents"
+    required: false
+    default: "left"
+    allowed_values: ["left", "center", "preserve"]
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A processor for DOCX files"

config/processors/globals/HTML_PROCESSOR/1.0.0.yaml

@@ -0,0 +1,49 @@
+version: "1.0.0"
+name: "HTML Processor"
+file_types: "html"
+description: "A processor for HTML files"
+configuration:
+  chunking_patterns:
+    name: "Chunking Patterns"
+    description: "A list of Patterns used to chunk files into logical pieces"
+    type: "chunking_patterns"
+    required: false
+  chunking_heading_level:
+    name: "Chunking Heading Level"
+    type: "integer"
+    description: "Maximum heading level to consider for chunking (1-6)"
+    required: false
+    default: 2
+  html_tags:
+    name: "HTML Tags"
+    type: "string"
+    description: "A comma-separated list of HTML tags"
+    required: true
+    default: "p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td"
+  html_end_tags:
+    name: "HTML End Tags"
+    type: "string"
+    description: "A comma-separated list of HTML end tags (where can the chunk end)"
+    required: true
+    default: "p, li, table"
+  html_included_elements:
+    name: "HTML Included Elements"
+    type: "string"
+    description: "A comma-separated list of elements to be included"
+    required: true
+    default: "article, main"
+  html_excluded_elements:
+    name: "HTML Excluded Elements"
+    type: "string"
+    description: "A comma-separated list of elements to be excluded"
+    required: false
+    default: "header, footer, nav, script"
+  html_excluded_classes:
+    name: "HTML Excluded Classes"
+    type: "string"
+    description: "A comma-separated list of classes to be excluded"
+    required: false
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A processor for HTML files"

config/processors/globals/MARKDOWN_PROCESSOR/1.0.0.yaml

@@ -0,0 +1,20 @@
+version: "1.0.0"
+name: "Markdown Processor"
+file_types: "md"
+description: "A Processor for markdown files"
+configuration:
+  chunking_patterns:
+    name: "Chunking Patterns"
+    description: "A list of Patterns used to chunk files into logical pieces"
+    type: "chunking_patterns"
+    required: false
+  chunking_heading_level:
+    name: "Chunking Heading Level"
+    type: "integer"
+    description: "Maximum heading level to consider for chunking (1-6)"
+    required: false
+    default: 2
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A Processor for markdown files"

config/processors/globals/PDF_PROCESSOR/1.0.0.yaml

@@ -0,0 +1,20 @@
+version: "1.0.0"
+name: "PDF Processor"
+file_types: "pdf"
+description: "A Processor for PDF files"
+configuration:
+  chunking_patterns:
+    name: "Chunking Patterns"
+    description: "A list of Patterns used to chunk files into logical pieces"
+    type: "chunking_patterns"
+    required: false
+  chunking_heading_level:
+    name: "Chunking Heading Level"
+    type: "integer"
+    description: "Maximum heading level to consider for chunking (1-6)"
+    required: false
+    default: 2
+metadata:
+  author: "System"
+  date_added: "2023-01-01"
+  description: "A Processor for PDF files"

config/type_defs/catalog_types.py

@@ -2,28 +2,10 @@
 CATALOG_TYPES = {
     "STANDARD_CATALOG": {
         "name": "Standard Catalog",
-        "Description": "A Catalog with information in Evie's Library, to be considered as a whole",
+        "description": "A Catalog with information in Evie's Library, to be considered as a whole",
-        "configuration": {},
-        "document_version_configurations": []
     },
     "DOSSIER_CATALOG": {
         "name": "Dossier Catalog",
-        "Description": "A Catalog with information in Evie's Library in which several Dossiers can be stored",
+        "description": "A Catalog with information in Evie's Library in which several Dossiers can be stored",
-        "configuration": {
-            "tagging_fields": {
-                "name": "Tagging Fields",
-                "type": "tagging_fields",
-                "description": """Define the metadata fields that will be used for tagging documents.
-                                Each field must have:
-                                - type: one of 'string', 'integer', 'float', 'date', 'enum'
-                                - required: boolean indicating if the field is mandatory
-                                - description: field description
-                                - allowed_values: list of values (for enum type only)
-                                - min_value/max_value: range limits (for numeric types only)""",
-                "required": True,
-                "default": {},
-            }
-        },
-        "document_version_configurations": ["tagging_fields"]
     },
 }

config/type_defs/processor_types.py

@@ -1,168 +1,28 @@
-# Catalog Types
+# Processor Types
 PROCESSOR_TYPES = {
     "HTML_PROCESSOR": {
         "name": "HTML Processor",
+        "description": "A processor for HTML files",
         "file_types": "html",
-        "Description": "A processor for HTML files",
-        "configuration": {
-            "chunking_patterns": {
-                "name": "Chunking Patterns",
-                "description": "A list of Patterns used to chunk files into logical pieces",
-                "type": "chunking_patterns",
-                "required": False
-            },
-            "chunking_heading_level": {
-                "name": "Chunking Heading Level",
-                "type": "integer",
-                "description": "Maximum heading level to consider for chunking (1-6)",
-                "required": False,
-                "default": 2
-            },
-            "html_tags": {
-                "name": "HTML Tags",
-                "type": "string",
-                "description": "A comma-separated list of HTML tags",
-                "required": True,
-                "default": "p, h1, h2, h3, h4, h5, h6, li, table, thead, tbody, tr, td"
-            },
-            "html_end_tags": {
-                "name": "HTML End Tags",
-                "type": "string",
-                "description": "A comma-separated list of HTML end tags (where can the chunk end)",
-                "required": True,
-                "default": "p, li, table"
-            },
-            "html_included_elements": {
-                "name": "HTML Included Elements",
-                "type": "string",
-                "description": "A comma-separated list of elements to be included",
-                "required": True,
-                "default": "article, main"
-            },
-            "html_excluded_elements": {
-                "name": "HTML Excluded Elements",
-                "type": "string",
-                "description": "A comma-separated list of elements to be excluded",
-                "required": False,
-                "default": "header, footer, nav, script"
-            },
-            "html_excluded_classes": {
-                "name": "HTML Excluded Classes",
-                "type": "string",
-                "description": "A comma-separated list of classes to be excluded",
-                "required": False,
-            },
-        },
     },
     "PDF_PROCESSOR": {
         "name": "PDF Processor",
+        "description": "A Processor for PDF files",
         "file_types": "pdf",
-        "Description": "A Processor for PDF files",
-        "configuration": {
-            "chunking_patterns": {
-                "name": "Chunking Patterns",
-                "description": "A list of Patterns used to chunk files into logical pieces",
-                "type": "chunking_patterns",
-                "required": False
-            },
-            "chunking_heading_level": {
-                "name": "Chunking Heading Level",
-                "type": "integer",
-                "description": "Maximum heading level to consider for chunking (1-6)",
-                "required": False,
-                "default": 2
-            },
-        },
     },
     "AUDIO_PROCESSOR": {
         "name": "AUDIO Processor",
+        "description": "A Processor for audio files",
         "file_types": "mp3, mp4, ogg",
-        "Description": "A Processor for audio files",
-        "configuration": {}
     },
     "MARKDOWN_PROCESSOR": {
         "name": "Markdown Processor",
+        "description": "A Processor for markdown files",
         "file_types": "md",
-        "Description": "A Processor for markdown files",
-        "configuration": {
-            "chunking_patterns": {
-                "name": "Chunking Patterns",
-                "description": "A list of Patterns used to chunk files into logical pieces",
-                "type": "chunking_patterns",
-                "required": False
-            },
-            "chunking_heading_level": {
-                "name": "Chunking Heading Level",
-                "type": "integer",
-                "description": "Maximum heading level to consider for chunking (1-6)",
-                "required": False,
-                "default": 2
-            },
-        }
     },
     "DOCX_PROCESSOR": {
         "name": "DOCX Processor",
+        "description": "A processor for DOCX files",
         "file_types": "docx",
-        "Description": "A processor for DOCX files",
-        "configuration": {
-            "chunking_patterns": {
-                "name": "Chunking Patterns",
-                "description": "A list of Patterns used to chunk files into logical pieces",
-                "type": "chunking_patterns",
-                "required": False
-            },
-            "chunking_heading_level": {
-                "name": "Chunking Heading Level",
-                "type": "integer",
-                "description": "Maximum heading level to consider for chunking (1-6)",
-                "required": False,
-                "default": 2
-            },
-            "extract_comments": {
-                "name": "Extract Comments",
-                "type": "boolean",
-                "description": "Whether to include document comments in the markdown",
-                "required": False,
-                "default": False
-            },
-            "extract_headers_footers": {
-                "name": "Extract Headers/Footers",
-                "type": "boolean",
-                "description": "Whether to include headers and footers in the markdown",
-                "required": False,
-                "default": False
-            },
-            "preserve_formatting": {
-                "name": "Preserve Formatting",
-                "type": "boolean",
-                "description": "Whether to preserve bold, italic, and other text formatting",
-                "required": False,
-                "default": True
-            },
-            "list_style": {
-                "name": "List Style",
-                "type": "enum",
-                "description": "How to format lists in markdown",
-                "required": False,
-                "default": "dash",
-                "allowed_values": ["dash", "asterisk", "plus"]
-            },
-            "image_handling": {
-                "name": "Image Handling",
-                "type": "enum",
-                "description": "How to handle embedded images",
-                "required": False,
-                "default": "skip",
-                "allowed_values": ["skip", "extract", "placeholder"]
-            },
-            "table_alignment": {
-                "name": "Table Alignment",
-                "type": "enum",
-                "description": "How to align table contents",
-                "required": False,
-                "default": "left",
-                "allowed_values": ["left", "center", "preserve"]
-            }
-        }
     }
 }

config/type_defs/specialist_types.py

@@ -16,5 +16,9 @@ SPECIALIST_TYPES = {
         "name": "Traicie Role Definition Specialist",
         "description": "Assistant Defining Competencies and KO Criteria",
         "partner": "traicie"
+    },
+    "TRAICIE_SELECTION_SPECIALIST": {
+        "name": "Traicie Selection Specialist",
+        "description": "Recruitment Selection Assistant",
     }
 }

eveai_app/templates/base.html

@@ -54,6 +54,7 @@
         <hr>
         {% include 'footer.html' %}
         {% include 'scripts.html' %}
+        {% include 'ordered_list_configs.html' %}
         {% block scripts %}{% endblock %}
     </body>
-</html>
+</html>

eveai_app/templates/eveai_ordered_list_editor.html

@@ -0,0 +1,231 @@
+<script type="module">
+window.EveAI = window.EveAI || {};
+window.EveAI.OrderedListEditors = {
+    instances: {},
+    initialize: function(containerId, data, listType, options = {}) {
+        console.log('Initializing OrderedListEditor for', containerId, 'with data', data, 'and listType', listType);
+        const container = document.getElementById(containerId);
+        if (!container || typeof container !== 'object' || !('classList' in container)) {
+            console.error(`Container with ID ${containerId} not found or not a valid element:`, container);
+            return null;
+        }
+
+        if (this.instances[containerId]) return this.instances[containerId];
+
+        if (typeof window.Tabulator !== 'function') {
+            console.error('Tabulator not loaded (window.Tabulator missing).');
+            container.innerHTML = `<div class="alert alert-danger p-3">
+                <strong>Error:</strong> Tabulator not loaded
+            </div>`;
+            return null;
+        }
+
+        // Get the list type configuration
+        const listTypeConfig = this._getListTypeConfig(listType);
+        if (!listTypeConfig) {
+            console.error(`List type configuration for ${listType} not found.`);
+            container.innerHTML = `<div class="alert alert-danger p-3">
+                <strong>Error:</strong> List type configuration for ${listType} not found
+            </div>`;
+            return null;
+        }
+
+        // Create column definitions from list type
+        const columns = this._createColumnsFromListType(listTypeConfig);
+
+        // Initialize Tabulator
+        try {
+            console.log('Creating Tabulator for', containerId);
+            const table = new Tabulator(container, {
+                data: data || [],
+                columns: columns,
+                layout: "fitColumns",
+                movableRows: true,
+                height: "400px",
+                ...options
+            });
+
+            console.log('Tabulator created for', containerId);
+            container.classList.add('tabulator-initialized');
+
+            // Add row button
+            const addRowBtn = document.createElement('button');
+            addRowBtn.className = 'btn btn-sm btn-primary mt-2';
+            addRowBtn.innerHTML = 'Add Row';
+            addRowBtn.addEventListener('click', () => {
+                const newRow = {};
+                // Create empty row with default values
+                Object.entries(listTypeConfig).forEach(([key, field]) => {
+                    newRow[key] = field.default || '';
+                });
+                table.addRow(newRow);
+                this._updateTextarea(containerId, table);
+            });
+            container.parentNode.insertBefore(addRowBtn, container.nextSibling);
+
+            // Store instance
+            this.instances[containerId] = {
+                table: table,
+                textarea: document.getElementById(containerId.replace('-editor', ''))
+            };
+
+            // Update textarea on data change
+            table.on("dataChanged", () => {
+                this._updateTextarea(containerId, table);
+            });
+
+            return table;
+        } catch (e) {
+            console.error(`Error initializing Tabulator for ${containerId}:`, e);
+            container.innerHTML = `<div class="alert alert-danger p-3">
+                <strong>Error initializing Tabulator:</strong><br>${e.message}
+            </div>`;
+            return null;
+        }
+    },
+
+    _updateTextarea: function(containerId, table) {
+        const instance = this.instances[containerId];
+        if (instance && instance.textarea) {
+            instance.textarea.value = JSON.stringify(table.getData());
+        }
+    },
+
+    _getListTypeConfig: function(listType) {
+        // This would need to be implemented to fetch the list type configuration
+        // Could be from a global variable set in the template or via an API call
+        return window.listTypeConfigs && window.listTypeConfigs[listType];
+    },
+
+    _createColumnsFromListType: function(listTypeConfig) {
+        const columns = [];
+
+        // Add drag handle column for row reordering
+        columns.push({
+            formatter: "handle",
+            headerSort: false,
+            frozen: true,
+            width: 30,
+            minWidth: 30
+        });
+
+        // Add columns for each field in the list type
+        Object.entries(listTypeConfig).forEach(([key, field]) => {
+            const column = {
+                title: field.name || key,
+                field: key,
+                tooltip: field.description
+            };
+
+            // Set editor based on field type
+            if (field.type === 'boolean') {
+                column.formatter = 'tickCross';
+                column.editor = 'tickCross';
+                column.hozAlign = 'center';
+            } else if (field.type === 'enum' && field.allowed_values) {
+                column.editor = 'select';
+                column.editorParams = {
+                    values: field.allowed_values
+                };
+            } else {
+                column.editor = 'input';
+            }
+
+            columns.push(column);
+        });
+
+        // Add delete button column
+        columns.push({
+            formatter: function(cell, formatterParams, onRendered) {
+                return "<button class='btn btn-sm btn-danger'><i class='material-icons'>delete</i></button>";
+            },
+            width: 40,
+            hozAlign: "center",
+            headerSort: false,
+            cellClick: function(e, cell) {
+                cell.getRow().delete();
+            }
+        });
+
+        return columns;
+    },
+
+    get: function(containerId) {
+        return this.instances[containerId] || null;
+    },
+
+    destroy: function(containerId) {
+        if (this.instances[containerId]) {
+            if (this.instances[containerId].table && typeof this.instances[containerId].table.destroy === 'function') {
+                this.instances[containerId].table.destroy();
+            }
+            delete this.instances[containerId];
+        }
+        const container = document.getElementById(containerId);
+        if (container) {
+            container.classList.remove('tabulator-initialized');
+            container.innerHTML = '';
+        }
+    }
+};
+
+document.addEventListener('DOMContentLoaded', function() {
+    // Initialize list type configurations
+    window.listTypeConfigs = window.listTypeConfigs || {};
+
+    // Initialize ordered list editors
+    document.querySelectorAll('.ordered-list-field').forEach(function(textarea) {
+        const containerId = textarea.id + '-editor';
+        const container = document.getElementById(containerId);
+        if (!container) return;
+
+        try {
+            const data = textarea.value ? JSON.parse(textarea.value) : [];
+            const listType = textarea.getAttribute('data-list-type');
+
+            // Check if we have the list type configuration
+            if (listType && !window.listTypeConfigs[listType]) {
+                console.warn(`List type configuration for ${listType} not found. The editor may not work correctly.`);
+            }
+
+            window.EveAI.OrderedListEditors.initialize(containerId, data, listType);
+        } catch (e) {
+            console.error('Error initializing ordered list editor:', e);
+            container.innerHTML = `<div class="alert alert-danger p-3">
+                <strong>Error initializing ordered list editor:</strong><br>${e.message}
+            </div>`;
+        }
+    });
+});
+</script>
+
+<style>
+/* Tabulator styling */
+.ordered-list-editor {
+    margin-bottom: 1rem;
+}
+
+/* Make sure the Tabulator container has a proper height */
+.ordered-list-editor .tabulator {
+    height: 400px;
+    width: 100%;
+    border: 1px solid #dee2e6;
+    border-radius: 0.25rem;
+}
+
+/* Style for the handle column */
+.ordered-list-editor .tabulator-row-handle {
+    cursor: move;
+}
+
+/* Style for the delete button */
+.ordered-list-editor .tabulator-cell button.btn-danger {
+    padding: 0.25rem 0.5rem;
+    font-size: 0.75rem;
+}
+
+/* Style for boolean columns */
+.ordered-list-editor .tabulator-cell[data-type="boolean"] {
+    text-align: center;
+}
+</style>

eveai_app/templates/macros.html

@@ -1,3 +1,12 @@
+{# Helper functie om veilig de class van een veld te krijgen #}
+{% macro get_field_class(field, default='') %}
+    {% if field.render_kw is not none and field.render_kw.get('class') is not none %}
+        {{ field.render_kw.get('class') }}
+    {% else %}
+        {{ default }}
+    {% endif %}
+{% endmacro %}
+
 {% macro render_field_content(field, disabled=False, readonly=False, class='') %}
     {% if field.type == 'BooleanField' %}
         <div class="form-group">
@@ -55,9 +64,13 @@
                 {% endif %}
             {% endif %}
 
+            {% set field_class = get_field_class(field) %}
             {% if field.type == 'TextAreaField' and 'json-editor' in class %}
                 <div id="{{ field.id }}-editor" class="json-editor-container"></div>
                 {{ field(class="form-control d-none " + class, disabled=disabled, readonly=readonly) }}
+            {% elif field.type == 'OrderedListField' or 'ordered-list-field' in field_class %}
+                {# Behoud ordered-list-field class en voeg form-control toe #}
+                {{ field(class="form-control " + field_class|trim, disabled=disabled, readonly=readonly) }}
             {% elif field.type == 'SelectField' %}
                 {{ field(class="form-control form-select " + class, disabled=disabled, readonly=readonly) }}
             {% else %}
@@ -76,6 +89,7 @@
 {% endmacro %}
 
 
+
 {% macro render_field(field, disabled_fields=[], readonly_fields=[], exclude_fields=[], class='') %}
     <!-- Debug info -->
     <!-- Field name: {{ field.name }}, Field type: {{ field.__class__.__name__ }} -->

eveai_app/templates/ordered_list_configs.html

@@ -0,0 +1,7 @@
+{# Include this template in any page that uses ordered_list fields #}
+{# Usage: {% include 'ordered_list_configs.html' %} #}
+{# The form must be available in the template context as 'form' #}
+
+{% if form and form.get_list_type_configs_js %}
+    {{ form.get_list_type_configs_js()|safe }}
+{% endif %}

eveai_app/templates/scripts.html

@@ -9,6 +9,7 @@
 <script src="{{url_for('static', filename='assets/js/material-kit-pro.min.js')}}"></script>
 
 {% include 'eveai_json_editor.html' %}
+{% include 'eveai_ordered_list_editor.html' %}
 
 <script>
 document.addEventListener('DOMContentLoaded', function() {

eveai_app/views/document_views.py

@@ -20,7 +20,6 @@ from common.utils.document_utils import create_document_stack, start_embedding_t
 from common.utils.dynamic_field_utils import create_default_config_from_type_config
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException, EveAIException
-from config.type_defs.processor_types import PROCESSOR_TYPES
 from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, \
     CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
 from common.utils.middleware import mw_before_request
@@ -29,7 +28,6 @@ from common.utils.nginx_utils import prefixed_url_for
 from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
 from .document_list_view import DocumentListView
 from .document_version_list_view import DocumentVersionListView
-from config.type_defs.catalog_types import CATALOG_TYPES
 
 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
 
@@ -126,8 +124,8 @@ def edit_catalog(catalog_id):
     tenant_id = session.get('tenant').get('id')
 
     form = EditCatalogForm(request.form, obj=catalog)
-    configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
+    full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
-    form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
+    form.add_dynamic_fields("configuration", full_config, catalog.configuration)
 
     if request.method == 'POST' and form.validate_on_submit():
         form.populate_obj(catalog)
@@ -160,8 +158,9 @@ def processor():
         new_processor = Processor()
         form.populate_obj(new_processor)
         new_processor.catalog_id = form.catalog.data.id
+        processor_config = cache_manager.processors_config_cache.get_config(new_processor.type)
         new_processor.configuration = create_default_config_from_type_config(
-            PROCESSOR_TYPES[new_processor.type]["configuration"])
+            processor_config["configuration"])
 
         set_logging_information(new_processor, dt.now(tz.utc))
 
@@ -197,8 +196,8 @@ def edit_processor(processor_id):
     # Create form instance with the processor
     form = EditProcessorForm(request.form, obj=processor)
 
-    configuration_config = PROCESSOR_TYPES[processor.type]["configuration"]
+    full_config = cache_manager.processors_config_cache.get_config(processor.type)
-    form.add_dynamic_fields("configuration", configuration_config, processor.configuration)
+    form.add_dynamic_fields("configuration", full_config, processor.configuration)
 
     if form.validate_on_submit():
         # Update basic fields
@@ -390,9 +389,10 @@ def add_document():
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+        document_version_configurations = full_config['document_version_configurations']
         for config in document_version_configurations:
-            form.add_dynamic_fields(config, catalog.configuration[config])
+            form.add_dynamic_fields(config, full_config, catalog.configuration[config])
 
     if form.validate_on_submit():
         try:
@@ -403,7 +403,8 @@ def add_document():
             filename = secure_filename(file.filename)
             extension = filename.rsplit('.', 1)[1].lower()
             catalog_properties = {}
-            document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+            full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+            document_version_configurations = full_config['document_version_configurations']
             for config in document_version_configurations:
                 catalog_properties[config] = form.get_dynamic_data(config)
 
@@ -445,9 +446,10 @@ def add_url():
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+        document_version_configurations = full_config['document_version_configurations']
         for config in document_version_configurations:
-            form.add_dynamic_fields(config, catalog.configuration[config])
+            form.add_dynamic_fields(config, full_config, catalog.configuration[config])
 
     if form.validate_on_submit():
         try:
@@ -459,7 +461,8 @@ def add_url():
             file_content, filename, extension = process_url(url, tenant_id)
 
             catalog_properties = {}
-            document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+            full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+            document_version_configurations = full_config['document_version_configurations']
             for config in document_version_configurations:
                 catalog_properties[config] = form.get_dynamic_data(config)
 
@@ -582,13 +585,14 @@ def edit_document_version_view(document_version_id):
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+        document_version_configurations = full_config['document_version_configurations']
         for config in document_version_configurations:
-            form.add_dynamic_fields(config, catalog.configuration[config], doc_vers.catalog_properties[config])
+            form.add_dynamic_fields(config, full_config, doc_vers.catalog_properties[config])
 
     if form.validate_on_submit():
         catalog_properties = {}
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        # Use the full_config variable we already defined
         for config in document_version_configurations:
             catalog_properties[config] = form.get_dynamic_data(config)
 
@@ -897,4 +901,3 @@ def clean_markdown(markdown):
     if markdown.endswith("```"):
         markdown = markdown[:-3].strip()
     return markdown
-

eveai_app/views/dynamic_form_base.py

@@ -49,6 +49,51 @@ class ChunkingPatternsField(TextAreaField):
         super().__init__(*args, **kwargs)
 
 
+class OrderedListField(TextAreaField):
+    """Field for ordered list data that will be rendered as a Tabulator table"""
+
+    def __init__(self, *args, **kwargs):
+        list_type = kwargs.pop('list_type', '')
+
+        # Behoud bestaande render_kw attributen als die er zijn
+        if 'render_kw' in kwargs:
+            existing_render_kw = kwargs['render_kw']
+        else:
+            existing_render_kw = {}
+
+        current_app.logger.debug(f"incomming render_kw for ordered list field: {existing_render_kw}")
+
+        # Stel nieuwe render_kw samen
+        new_render_kw = {
+            'data-list-type': list_type,
+            'data-handle-enter': 'true'
+        }
+
+        # Voeg klasse toe en behoud bestaande klassen
+        if 'class' in existing_render_kw:
+            existing_classes = existing_render_kw['class']
+            if isinstance(existing_classes, list):
+                existing_classes += ' ordered-list-field'
+                new_render_kw['class'] = existing_classes
+            else:
+                # String classes samenvoegen
+                new_render_kw['class'] = f"{existing_classes} ordered-list-field"
+        else:
+            new_render_kw['class'] = 'ordered-list-field'
+
+        # Voeg alle bestaande attributen toe aan nieuwe render_kw
+        for key, value in existing_render_kw.items():
+            if key != 'class':  # Klassen hebben we al verwerkt
+                new_render_kw[key] = value
+
+        current_app.logger.debug(f"final render_kw for ordered list field: {new_render_kw}")
+
+        # Update kwargs met de nieuwe gecombineerde render_kw
+        kwargs['render_kw'] = new_render_kw
+
+        super().__init__(*args, **kwargs)
+
+
 class DynamicFormBase(FlaskForm):
     def __init__(self, formdata=None, *args, **kwargs):
         super(DynamicFormBase, self).__init__(*args, **kwargs)
@@ -89,6 +134,8 @@ class DynamicFormBase(FlaskForm):
             validators_list.append(self._validate_tagging_fields_filter)
         elif field_type == 'dynamic_arguments':
             validators_list.append(self._validate_dynamic_arguments)
+        elif field_type == 'ordered_list':
+            validators_list.append(self._validate_ordered_list)
 
         return validators_list
 
@@ -227,10 +274,52 @@ class DynamicFormBase(FlaskForm):
         except Exception as e:
             raise ValidationError(f"Invalid argument definition: {str(e)}")
 
+    def _validate_ordered_list(self, form, field):
+        """Validate the ordered list structure"""
+        if not field.data:
+            return
+
+        try:
+            # Parse JSON data
+            list_data = json.loads(field.data)
+
+            # Validate it's a list
+            if not isinstance(list_data, list):
+                raise ValidationError("Ordered list must be a list")
+
+            # Validate each item in the list is a dictionary
+            for i, item in enumerate(list_data):
+                if not isinstance(item, dict):
+                    raise ValidationError(f"Item {i} in ordered list must be an object")
+
+        except json.JSONDecodeError:
+            raise ValidationError("Invalid JSON format")
+        except Exception as e:
+            raise ValidationError(f"Invalid ordered list: {str(e)}")
+
     def add_dynamic_fields(self, collection_name, config, initial_data=None):
-        """Add dynamic fields to the form based on the configuration."""
+        """Add dynamic fields to the form based on the configuration.
+
+        Args:
+            collection_name: The name of the collection of fields to add
+            config: The full configuration object, which should contain the field definitions
+                   for the collection_name and may also contain list_type definitions
+            initial_data: Optional initial data for the fields
+        """
+        current_app.logger.debug(f"Adding dynamic fields for collection {collection_name} with config: {config}")
+        # Store the full configuration for later use in get_list_type_configs_js
+        if not hasattr(self, '_full_configs'):
+            self._full_configs = {}
+        self._full_configs[collection_name] = config
+
+        # Get the specific field configuration for this collection
+        field_config = config.get(collection_name, {})
+        if not field_config:
+            # Handle the case where config is already the specific field configuration
+            return
+
         self.dynamic_fields[collection_name] = []
-        for field_name, field_def in config.items():
+        for field_name, field_def in field_config.items():
             # Prefix the field name with the collection name
             full_field_name = f"{collection_name}_{field_name}"
             label = field_def.get('name', field_name)
@@ -264,6 +353,12 @@ class DynamicFormBase(FlaskForm):
                 field_class = ChunkingPatternsField
                 extra_classes = ['monospace-text', 'pattern-input']
                 field_kwargs = {}
+            elif field_type == 'ordered_list':
+                current_app.logger.debug(f"Adding ordered list field for {full_field_name}")
+                field_class = OrderedListField
+                extra_classes = ''
+                list_type = field_def.get('list_type', '')
+                field_kwargs = {'list_type': list_type}
             else:
                 extra_classes = ''
                 field_class = {
@@ -289,6 +384,12 @@ class DynamicFormBase(FlaskForm):
                     except (TypeError, ValueError) as e:
                         current_app.logger.error(f"Error converting initial data to JSON: {e}")
                         field_data = "{}"
+                elif field_type == 'ordered_list' and isinstance(field_data, list):
+                    try:
+                        field_data = json.dumps(field_data)
+                    except (TypeError, ValueError) as e:
+                        current_app.logger.error(f"Error converting ordered list data to JSON: {e}")
+                        field_data = "[]"
                 elif field_type == 'chunking_patterns':
                     try:
                         field_data = json_to_patterns(field_data)
@@ -305,6 +406,9 @@ class DynamicFormBase(FlaskForm):
                 render_kw['data-bs-toggle'] = 'tooltip'
                 render_kw['data-bs-placement'] = 'right'
 
+
+            current_app.logger.debug(f"render_kw for {full_field_name}: {render_kw}")
+
             # Create the field
             field_kwargs.update({
                 'label': label,
@@ -340,6 +444,73 @@ class DynamicFormBase(FlaskForm):
         # Return all fields that are not dynamic
         return [field for name, field in self._fields.items() if name not in dynamic_field_names]
 
+    def get_list_type_configs_js(self):
+        """Generate JavaScript code for list type configurations used by ordered_list fields."""
+        from common.extensions import cache_manager
+
+        list_types = {}
+
+        # First check if we have any full configurations stored
+        if hasattr(self, '_full_configs'):
+            # Look for list types in the stored full configurations
+            for config_name, config in self._full_configs.items():
+                for key, value in config.items():
+                    # Check if this is a list type definition (not a field definition)
+                    if isinstance(value, dict) and all(isinstance(v, dict) for v in value.values()):
+                        # This looks like a list type definition
+                        list_types[key] = value
+
+        # Collect all list types used in ordered_list fields
+        for collection_name, field_names in self.dynamic_fields.items():
+            for full_field_name in field_names:
+                field = getattr(self, full_field_name)
+                if isinstance(field, OrderedListField):
+                    list_type = field.render_kw.get('data-list-type')
+                    if list_type and list_type not in list_types:
+                        # First try to get from current_app.config
+                        list_type_config = current_app.config.get('LIST_TYPES', {}).get(list_type)
+                        if list_type_config:
+                            list_types[list_type] = list_type_config
+                        else:
+                            # Try to find the list type in specialist configurations using the cache
+                            try:
+                                # Get all specialist types
+                                specialist_types = cache_manager.specialists_types_cache.get_types()
+
+                                # For each specialist type, check if it has the list type we're looking for
+                                for specialist_type in specialist_types:
+                                    try:
+                                        # Get the latest version for this specialist type
+                                        latest_version = cache_manager.specialists_version_tree_cache.get_latest_version(specialist_type)
+
+                                        # Get the configuration for this specialist type and version
+                                        specialist_config = cache_manager.specialists_config_cache.get_config(specialist_type, latest_version)
+
+                                        # Check if this specialist has the list type we're looking for
+                                        if list_type in specialist_config:
+                                            list_types[list_type] = specialist_config[list_type]
+                                            break
+                                    except Exception as e:
+                                        current_app.logger.debug(f"Error checking specialist {specialist_type}: {e}")
+                                        continue
+                            except Exception as e:
+                                current_app.logger.error(f"Error retrieving specialist configurations: {e}")
+
+        # If no list types found, return empty script
+        if not list_types:
+            return ""
+
+        # Generate JavaScript code
+        js_code = "<script>\n"
+        js_code += "window.listTypeConfigs = window.listTypeConfigs || {};\n"
+
+        for list_type, config in list_types.items():
+            js_code += f"window.listTypeConfigs['{list_type}'] = {json.dumps(config, indent=2)};\n"
+
+        js_code += "</script>\n"
+
+        return js_code
+
     def get_dynamic_fields(self):
         """Return a dictionary of dynamic fields per collection."""
         result = {}
@@ -361,7 +532,7 @@ class DynamicFormBase(FlaskForm):
             if field.type == 'BooleanField':
                 data[original_field_name] = full_field_name in self.raw_formdata
                 current_app.logger.debug(f"Value for {original_field_name} is {data[original_field_name]}")
-            elif isinstance(field, (TaggingFieldsField, TaggingFieldsFilterField, DynamicArgumentsField)) and field.data:
+            elif isinstance(field, (TaggingFieldsField, TaggingFieldsFilterField, DynamicArgumentsField, OrderedListField)) and field.data:
                 try:
                     data[original_field_name] = json.loads(field.data)
                 except json.JSONDecodeError:
@@ -443,4 +614,4 @@ def validate_tagging_fields(form, field):
     except json.JSONDecodeError:
         raise ValidationError("Invalid JSON format")
     except (TypeError, ValueError) as e:
-        raise ValidationError(f"Invalid field definition: {str(e)}")
+        raise ValidationError(f"Invalid field definition: {str(e)}")

eveai_app/views/interaction_views.py

@@ -204,8 +204,7 @@ def edit_specialist(specialist_id):
     form = EditSpecialistForm(request.form, obj=specialist)
 
     specialist_config = cache_manager.specialists_config_cache.get_config(specialist.type, specialist.type_version)
-    configuration_config = specialist_config.get('configuration')
+    form.add_dynamic_fields("configuration", specialist_config, specialist.configuration)
-    form.add_dynamic_fields("configuration", configuration_config, specialist.configuration)
 
     agent_rows = prepare_table_for_macro(specialist.agents,
                                          [('id', ''), ('name', ''), ('type', ''), ('type_version', '')])
@@ -521,8 +520,7 @@ def edit_asset_version(asset_version_id):
     form = EditEveAIAssetVersionForm(asset_version)
     asset_config = cache_manager.assets_config_cache.get_config(asset_version.asset.type,
                                                                 asset_version.asset.type_version)
-    configuration_config = asset_config.get('configuration')
+    form.add_dynamic_fields("configuration", asset_config, asset_version.configuration)
-    form.add_dynamic_fields("configuration", configuration_config, asset_version.configuration)
 
     if form.validate_on_submit():
         # Update the configuration dynamic fields
@@ -582,9 +580,8 @@ def execute_specialist(specialist_id):
         return redirect(prefixed_url_for('interaction_bp.specialists'))
 
     form = ExecuteSpecialistForm(request.form, obj=specialist)
-    arguments_config = specialist_config.get('arguments', None)
+    if 'arguments' in specialist_config:
-    if arguments_config:
+        form.add_dynamic_fields('arguments', specialist_config)
-        form.add_dynamic_fields('arguments', arguments_config)
 
     if form.validate_on_submit():
         # We're only interested in gathering the dynamic arguments
@@ -674,4 +671,4 @@ def session_interactions(chat_session_id):
     This route shows all interactions for a given chat_session_id (int).
     """
     chat_session = ChatSession.query.get_or_404(chat_session_id)
-    return session_interactions_by_session_id(chat_session.session_id)
+    return session_interactions_by_session_id(chat_session.session_id)

nginx/frontend_src/js/main.js

@@ -42,5 +42,8 @@ import { createJSONEditor } from 'vanilla-jsoneditor';
 // Maak de factory functie globaal beschikbaar als je dit elders in je code gebruikt.
 window.createJSONEditor = createJSONEditor;
 
+import { Tabulator } from 'tabulator-tables';
+window.Tabulator = Tabulator;
+
 // Eventueel een log om te bevestigen dat de bundel is geladen
 console.log('JavaScript bibliotheken gebundeld en geladen via main.js.');

nginx/package-lock.json

@@ -10,6 +10,7 @@
         "datatables.net": "^2.3.1",
         "jquery": "^3.7.1",
         "select2": "^4.1.0-rc.0",
+        "tabulator-tables": "^6.3.1",
         "vanilla-jsoneditor": "^3.5.0"
       },
       "devDependencies": {
@@ -3648,6 +3649,12 @@
         "node": ">=18"
       }
     },
+    "node_modules/tabulator-tables": {
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/tabulator-tables/-/tabulator-tables-6.3.1.tgz",
+      "integrity": "sha512-qFW7kfadtcaISQIibKAIy0f3eeIXUVi8242Vly1iJfMD79kfEGzfczNuPBN/80hDxHzQJXYbmJ8VipI40hQtfA==",
+      "license": "MIT"
+    },
     "node_modules/term-size": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz",

nginx/package.json

@@ -5,6 +5,7 @@
     "datatables.net": "^2.3.1",
     "jquery": "^3.7.1",
     "select2": "^4.1.0-rc.0",
+    "tabulator-tables": "^6.3.1",
     "vanilla-jsoneditor": "^3.5.0"
   },
   "devDependencies": {