- Allowing for multiple types of Catalogs

- Introduction of retrievers
- Ensuring processing information is collected from Catalog iso Tenant
- Introduction of a generic Form class to enable dynamic fields based on a configuration
- Realisation of Retriever functionality to support dynamic fields

eveai_app/views/document_forms.py

@@ -1,4 +1,4 @@
-from flask import session, current_app
+from flask import session, current_app, request
 from flask_wtf import FlaskForm
 from wtforms import (StringField, BooleanField, SubmitField, DateField, IntegerField, FloatField, SelectMultipleField,
                      SelectField, FieldList, FormField, TextAreaField, URLField)
@@ -6,6 +6,14 @@ from wtforms.validators import DataRequired, Length, Optional, URL, ValidationEr
 from flask_wtf.file import FileField, FileAllowed, FileRequired
 import json
 
+from wtforms_sqlalchemy.fields import QuerySelectField
+
+from common.models.document import Catalog
+
+from config.catalog_types import CATALOG_TYPES
+from config.retriever_types import RETRIEVER_TYPES
+from .dynamic_form_base import DynamicFormBase
+
 
 def allowed_file(form, field):
     if field.data:
@@ -26,6 +34,23 @@ def validate_json(form, field):
 class CatalogForm(FlaskForm):
     name = StringField('Name', validators=[DataRequired(), Length(max=50)])
     description = TextAreaField('Description', validators=[Optional()])
+    # Parent ID (Optional for root-level catalogs)
+    parent = QuerySelectField(
+        'Parent Catalog',
+        query_factory=lambda: Catalog.query.all(),
+        allow_blank=True,
+        get_label='name',
+        validators=[Optional()],
+    )
+
+    # Select Field for Catalog Type (Uses the CATALOG_TYPES defined in config)
+    type = SelectField('Catalog Type', validators=[DataRequired()])
+
+    # Metadata fields
+    user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
+    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
+    configuration = TextAreaField('Configuration', validators=[Optional(), validate_json])
+
     # HTML Embedding Variables
     html_tags = StringField('HTML Tags', validators=[DataRequired()],
                             default='p, h1, h2, h3, h4, h5, h6, li, , tbody, tr, td')
@@ -38,19 +63,65 @@ class CatalogForm(FlaskForm):
                                   default=2000)
     max_chunk_size = IntegerField('Maximum Chunk Size (3000)', validators=[NumberRange(min=0), Optional()],
                                   default=3000)
-    # Embedding Search variables
-    es_k = IntegerField('Limit for Searching Embeddings (5)',
-                        default=5,
-                        validators=[NumberRange(min=0)])
-    es_similarity_threshold = FloatField('Similarity Threshold for Searching Embeddings (0.5)',
-                                         default=0.5,
-                                         validators=[NumberRange(min=0, max=1)])
     # Chat Variables
     chat_RAG_temperature = FloatField('RAG Temperature', default=0.3, validators=[NumberRange(min=0, max=1)])
     chat_no_RAG_temperature = FloatField('No RAG Temperature', default=0.5, validators=[NumberRange(min=0, max=1)])
     # Tuning variables
     embed_tuning = BooleanField('Enable Embedding Tuning', default=False)
-    rag_tuning = BooleanField('Enable RAG Tuning', default=False)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Dynamically populate the 'type' field using the constructor
+        self.type.choices = [(key, value['name']) for key, value in CATALOG_TYPES.items()]
+
+
+class RetrieverForm(FlaskForm):
+    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
+    description = TextAreaField('Description', validators=[Optional()])
+    # Catalog for the Retriever
+    catalog = QuerySelectField(
+        'Catalog ID',
+        query_factory=lambda: Catalog.query.all(),
+        allow_blank=True,
+        get_label='name',
+        validators=[Optional()],
+    )
+    # Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config)
+    type = SelectField('Retriever Type', validators=[DataRequired()])
+
+    # Metadata fields
+    user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
+    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Dynamically populate the 'type' field using the constructor
+        self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
+
+
+class EditRetrieverForm(DynamicFormBase):
+    name = StringField('Name', validators=[DataRequired(), Length(max=50)])
+    description = TextAreaField('Description', validators=[Optional()])
+    # Catalog for the Retriever
+    catalog = QuerySelectField(
+        'Catalog ID',
+        query_factory=lambda: Catalog.query.all(),
+        allow_blank=True,
+        get_label='name',
+        validators=[Optional()],
+    )
+    # Select Field for Retriever Type (Uses the RETRIEVER_TYPES defined in config)
+    type = SelectField('Retriever Type', validators=[DataRequired()], render_kw={'readonly': True})
+
+    # Metadata fields
+    user_metadata = TextAreaField('User Metadata', validators=[Optional(), validate_json])
+    system_metadata = TextAreaField('System Metadata', validators=[Optional(), validate_json])
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Set the retriever type choices (loaded from config)
+        self.type.choices = [(key, value['name']) for key, value in RETRIEVER_TYPES.items()]
 
 
 class AddDocumentForm(FlaskForm):