- Replace old implementation of PROCESSOR_TYPES and CATALOG_TYPES with the new cached approach

- Add an ordered_list dynamic field type (to be refined)
- Add tabulator javascript library to project

eveai_app/views/document_views.py

@@ -20,7 +20,6 @@ from common.utils.document_utils import create_document_stack, start_embedding_t
 from common.utils.dynamic_field_utils import create_default_config_from_type_config
 from common.utils.eveai_exceptions import EveAIInvalidLanguageException, EveAIUnsupportedFileType, \
     EveAIDoubleURLException, EveAIException
-from config.type_defs.processor_types import PROCESSOR_TYPES
 from .document_forms import AddDocumentForm, AddURLForm, EditDocumentForm, EditDocumentVersionForm, \
     CatalogForm, EditCatalogForm, RetrieverForm, EditRetrieverForm, ProcessorForm, EditProcessorForm
 from common.utils.middleware import mw_before_request
@@ -29,7 +28,6 @@ from common.utils.nginx_utils import prefixed_url_for
 from common.utils.view_assistants import form_validation_failed, prepare_table_for_macro
 from .document_list_view import DocumentListView
 from .document_version_list_view import DocumentVersionListView
-from config.type_defs.catalog_types import CATALOG_TYPES
 
 document_bp = Blueprint('document_bp', __name__, url_prefix='/document')
 
@@ -126,8 +124,8 @@ def edit_catalog(catalog_id):
     tenant_id = session.get('tenant').get('id')
 
     form = EditCatalogForm(request.form, obj=catalog)
-    configuration_config = CATALOG_TYPES[catalog.type]["configuration"]
-    form.add_dynamic_fields("configuration", configuration_config, catalog.configuration)
+    full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+    form.add_dynamic_fields("configuration", full_config, catalog.configuration)
 
     if request.method == 'POST' and form.validate_on_submit():
         form.populate_obj(catalog)
@@ -160,8 +158,9 @@ def processor():
         new_processor = Processor()
         form.populate_obj(new_processor)
         new_processor.catalog_id = form.catalog.data.id
+        processor_config = cache_manager.processors_config_cache.get_config(new_processor.type)
         new_processor.configuration = create_default_config_from_type_config(
-            PROCESSOR_TYPES[new_processor.type]["configuration"])
+            processor_config["configuration"])
 
         set_logging_information(new_processor, dt.now(tz.utc))
 
@@ -197,8 +196,8 @@ def edit_processor(processor_id):
     # Create form instance with the processor
     form = EditProcessorForm(request.form, obj=processor)
 
-    configuration_config = PROCESSOR_TYPES[processor.type]["configuration"]
-    form.add_dynamic_fields("configuration", configuration_config, processor.configuration)
+    full_config = cache_manager.processors_config_cache.get_config(processor.type)
+    form.add_dynamic_fields("configuration", full_config, processor.configuration)
 
     if form.validate_on_submit():
         # Update basic fields
@@ -390,9 +389,10 @@ def add_document():
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+        document_version_configurations = full_config['document_version_configurations']
         for config in document_version_configurations:
-            form.add_dynamic_fields(config, catalog.configuration[config])
+            form.add_dynamic_fields(config, full_config, catalog.configuration[config])
 
     if form.validate_on_submit():
         try:
@@ -403,7 +403,8 @@ def add_document():
             filename = secure_filename(file.filename)
             extension = filename.rsplit('.', 1)[1].lower()
             catalog_properties = {}
-            document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+            full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+            document_version_configurations = full_config['document_version_configurations']
             for config in document_version_configurations:
                 catalog_properties[config] = form.get_dynamic_data(config)
 
@@ -445,9 +446,10 @@ def add_url():
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+        document_version_configurations = full_config['document_version_configurations']
         for config in document_version_configurations:
-            form.add_dynamic_fields(config, catalog.configuration[config])
+            form.add_dynamic_fields(config, full_config, catalog.configuration[config])
 
     if form.validate_on_submit():
         try:
@@ -459,7 +461,8 @@ def add_url():
             file_content, filename, extension = process_url(url, tenant_id)
 
             catalog_properties = {}
-            document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+            full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+            document_version_configurations = full_config['document_version_configurations']
             for config in document_version_configurations:
                 catalog_properties[config] = form.get_dynamic_data(config)
 
@@ -582,13 +585,14 @@ def edit_document_version_view(document_version_id):
 
     catalog = Catalog.query.get_or_404(catalog_id)
     if catalog.configuration and len(catalog.configuration) > 0:
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        full_config = cache_manager.catalogs_config_cache.get_config(catalog.type)
+        document_version_configurations = full_config['document_version_configurations']
         for config in document_version_configurations:
-            form.add_dynamic_fields(config, catalog.configuration[config], doc_vers.catalog_properties[config])
+            form.add_dynamic_fields(config, full_config, doc_vers.catalog_properties[config])
 
     if form.validate_on_submit():
         catalog_properties = {}
-        document_version_configurations = CATALOG_TYPES[catalog.type]['document_version_configurations']
+        # Use the full_config variable we already defined
         for config in document_version_configurations:
             catalog_properties[config] = form.get_dynamic_data(config)
 
@@ -897,4 +901,3 @@ def clean_markdown(markdown):
     if markdown.endswith("```"):
         markdown = markdown[:-3].strip()
     return markdown
-