- Introduction of dynamic Retrievers & Specialists

- Introduction of dynamic Processors
- Introduction of caching system
- Introduction of a better template manager
- Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists
- Start adaptation of chat client

eveai_workers/processors/base_processor.py

@@ -0,0 +1,88 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+
+from flask import current_app
+from common.extensions import minio_client
+from config.logging_config import TuningLogger
+
+
+class BaseProcessor(ABC):
+    def __init__(self, tenant, model_variables, document_version, catalog, processor):
+        self.tenant = tenant
+        self.model_variables = model_variables
+        self.document_version = document_version
+        self.catalog = catalog
+        self.processor = processor
+        self.tuning = processor.tuning if processor else False
+        self.tuning_logger = None
+        self._setup_tuning_logger()
+
+        self._log_tuning("Processor initialized", {
+            "processor_type": processor.type if processor else None,
+            "document_version": document_version.id if document_version else None,
+            "catalog": catalog.id if catalog else None
+        })
+
+    def _setup_tuning_logger(self):
+        try:
+            self.tuning_logger = TuningLogger(
+                'tuning',
+                tenant_id=self.tenant.id if self.tenant else None,
+                catalog_id=self.catalog.id if self.catalog else None,
+                processor_id=self.processor.id if self.processor else None,
+            )
+            # Verify logger is working with a test message
+            if self.tuning:
+                self.tuning_logger.log_tuning('processor', "Tuning logger initialized")
+        except Exception as e:
+            current_app.logger.error(f"Failed to setup tuning logger: {str(e)}")
+            raise
+
+    @abstractmethod
+    def process(self):
+        pass
+
+    def _save_markdown(self, markdown):
+        markdown_filename = f"{self.document_version.id}.md"
+        minio_client.upload_document_file(
+            self.tenant.id,
+            self.document_version.doc_id,
+            self.document_version.language,
+            self.document_version.id,
+            markdown_filename,
+            markdown.encode('utf-8')
+        )
+
+    def _log(self, message, level='debug'):
+        logger = current_app.logger
+        log_method = getattr(logger, level)
+        log_method(
+            f"{self.__class__.__name__} - Tenant {self.tenant.id}, Document {self.document_version.id}: {message}")
+
+    def _save_intermediate(self, content, filename):
+        minio_client.upload_document_file(
+            self.tenant.id,
+            self.document_version.doc_id,
+            self.document_version.language,
+            self.document_version.id,
+            filename,
+            content.encode('utf-8')
+        )
+
+    def _clean_markdown(self, markdown):
+        markdown = markdown.strip()
+        if markdown.startswith("```markdown"):
+            markdown = markdown[len("```markdown"):].strip()
+        if markdown.endswith("```"):
+            markdown = markdown[:-3].strip()
+
+        return markdown
+
+    def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None:
+        if self.tuning and self.tuning_logger:
+            try:
+                self.tuning_logger.log_tuning('processor', message, data)
+            except Exception as e:
+                current_app.logger.error(f"Processor: Error in tuning logging: {e}")
+
+