1807435339
- Introduction of dynamic Processors - Introduction of caching system - Introduction of a better template manager - Adaptation of ModelVariables to support dynamic Processors / Retrievers / Specialists - Start adaptation of chat client
89 lines
3.1 KiB
Python
89 lines
3.1 KiB
Python
from abc import ABC, abstractmethod
|
|
from typing import Dict, Any
|
|
|
|
from flask import current_app
|
|
from common.extensions import minio_client
|
|
from config.logging_config import TuningLogger
|
|
|
|
|
|
class BaseProcessor(ABC):
|
|
def __init__(self, tenant, model_variables, document_version, catalog, processor):
|
|
self.tenant = tenant
|
|
self.model_variables = model_variables
|
|
self.document_version = document_version
|
|
self.catalog = catalog
|
|
self.processor = processor
|
|
self.tuning = processor.tuning if processor else False
|
|
self.tuning_logger = None
|
|
self._setup_tuning_logger()
|
|
|
|
self._log_tuning("Processor initialized", {
|
|
"processor_type": processor.type if processor else None,
|
|
"document_version": document_version.id if document_version else None,
|
|
"catalog": catalog.id if catalog else None
|
|
})
|
|
|
|
def _setup_tuning_logger(self):
|
|
try:
|
|
self.tuning_logger = TuningLogger(
|
|
'tuning',
|
|
tenant_id=self.tenant.id if self.tenant else None,
|
|
catalog_id=self.catalog.id if self.catalog else None,
|
|
processor_id=self.processor.id if self.processor else None,
|
|
)
|
|
# Verify logger is working with a test message
|
|
if self.tuning:
|
|
self.tuning_logger.log_tuning('processor', "Tuning logger initialized")
|
|
except Exception as e:
|
|
current_app.logger.error(f"Failed to setup tuning logger: {str(e)}")
|
|
raise
|
|
|
|
@abstractmethod
|
|
def process(self):
|
|
pass
|
|
|
|
def _save_markdown(self, markdown):
|
|
markdown_filename = f"{self.document_version.id}.md"
|
|
minio_client.upload_document_file(
|
|
self.tenant.id,
|
|
self.document_version.doc_id,
|
|
self.document_version.language,
|
|
self.document_version.id,
|
|
markdown_filename,
|
|
markdown.encode('utf-8')
|
|
)
|
|
|
|
def _log(self, message, level='debug'):
|
|
logger = current_app.logger
|
|
log_method = getattr(logger, level)
|
|
log_method(
|
|
f"{self.__class__.__name__} - Tenant {self.tenant.id}, Document {self.document_version.id}: {message}")
|
|
|
|
def _save_intermediate(self, content, filename):
|
|
minio_client.upload_document_file(
|
|
self.tenant.id,
|
|
self.document_version.doc_id,
|
|
self.document_version.language,
|
|
self.document_version.id,
|
|
filename,
|
|
content.encode('utf-8')
|
|
)
|
|
|
|
def _clean_markdown(self, markdown):
|
|
markdown = markdown.strip()
|
|
if markdown.startswith("```markdown"):
|
|
markdown = markdown[len("```markdown"):].strip()
|
|
if markdown.endswith("```"):
|
|
markdown = markdown[:-3].strip()
|
|
|
|
return markdown
|
|
|
|
def _log_tuning(self, message: str, data: Dict[str, Any] = None) -> None:
|
|
if self.tuning and self.tuning_logger:
|
|
try:
|
|
self.tuning_logger.log_tuning('processor', message, data)
|
|
except Exception as e:
|
|
current_app.logger.error(f"Processor: Error in tuning logging: {e}")
|
|
|
|
|