From f5c9542a49c8d41112fa1f12e2e347274b864812 Mon Sep 17 00:00:00 2001 From: Josako Date: Tue, 24 Jun 2025 14:15:36 +0200 Subject: [PATCH] - Introducing translation service prompts - Ensure Traicie Role Definition Specialist complies to latest technical requirements - Ensure that empty historical messages do not cause a crash in eveai_client - take into account empty customisation options - make was not processed in the system dynamic attribute tenant_make - ensure only relevant makes are shown when creating magic links - refresh partner info when editing or adding Partner Services$ --- common/models/user.py | 24 +++++++++++++++++ common/utils/chat_utils.py | 25 ++++++++++++++---- common/utils/translation_utils.py | 21 +++++++++++++++ config/config.py | 7 ++--- .../translation_with_context/1.0.0.yaml | 15 +++++++++++ .../translation_without_context/1.0.0.yaml | 12 +++++++++ config/type_defs/prompt_types.py | 8 ++++++ eveai_app/views/dynamic_form_base.py | 7 +++-- eveai_app/views/interaction_forms.py | 4 ++- eveai_app/views/interaction_views.py | 9 ++++--- eveai_app/views/partner_views.py | 7 ++++- .../static/assets/js/chat-app.js | 26 ++++++++++--------- eveai_chat_client/views/chat_views.py | 8 +++++- .../TRAICIE_ROLE_DEFINITION_SPECIALIST/1_3.py | 3 +++ requirements.txt | 3 ++- 15 files changed, 147 insertions(+), 32 deletions(-) create mode 100644 common/utils/translation_utils.py create mode 100644 config/prompts/globals/translation_with_context/1.0.0.yaml create mode 100644 config/prompts/globals/translation_without_context/1.0.0.yaml diff --git a/common/models/user.py b/common/models/user.py index 75bcb23..ca1a9ab 100644 --- a/common/models/user.py +++ b/common/models/user.py @@ -317,3 +317,27 @@ class SpecialistMagicLinkTenant(db.Model): magic_link_code = db.Column(db.String(55), primary_key=True) tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False) + + +class TranslationCache(db.Model): + __bind_key__ = 'public' + __table_args__ = {'schema': 'public'} + + cache_key = db.Column(db.String(16), primary_key=True) + source_text = db.Column(db.Text, nullable=False) + translated_text = db.Column(db.Text, nullable=False) + source_language = db.Column(db.String(2), nullable=False) + target_language = db.Column(db.String(2), nullable=False) + context = db.Column(db.Text, nullable=True) + + # Translation cost + input_tokens = db.Column(db.Integer, nullable=False) + output_tokens = db.Column(db.Integer, nullable=False) + + # Tracking + created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now()) + created_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now(), onupdate=db.func.now()) + updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True) + + last_used_at = db.Column(db.DateTime, nullable=True) diff --git a/common/utils/chat_utils.py b/common/utils/chat_utils.py index 10783db..4a3fe32 100644 --- a/common/utils/chat_utils.py +++ b/common/utils/chat_utils.py @@ -1,14 +1,18 @@ +import json + """ Utility functions for chat customization. """ +from flask import current_app + def get_default_chat_customisation(tenant_customisation=None): """ Get chat customization options with default values for missing options. Args: - tenant_customization (dict, optional): The tenant's customization options. - Defaults to None. + tenant_customisation (dict or str, optional): The tenant's customization options. + Defaults to None. Can be a dict or a JSON string. Returns: dict: A dictionary containing all customization options with default values @@ -37,9 +41,20 @@ def get_default_chat_customisation(tenant_customisation=None): # Start with the default customization customisation = default_customisation.copy() + # Convert JSON string to dict if needed + if isinstance(tenant_customisation, str): + try: + tenant_customisation = json.loads(tenant_customisation) + current_app.logger.debug(f"Converted JSON string to dict: {tenant_customisation}") + except json.JSONDecodeError as e: + current_app.logger.error(f"Error parsing JSON customisation: {e}") + return default_customisation + # Update with tenant customization - for key, value in tenant_customisation.items(): - if key in customisation: - customisation[key] = value + current_app.logger.debug(f"Tenant customisation - in default creation: {tenant_customisation}") + if tenant_customisation: + for key, value in tenant_customisation.items(): + if key in customisation: + customisation[key] = value return customisation diff --git a/common/utils/translation_utils.py b/common/utils/translation_utils.py new file mode 100644 index 0000000..67ddafe --- /dev/null +++ b/common/utils/translation_utils.py @@ -0,0 +1,21 @@ +import xxhash +import json + +from common.utils.model_utils import get_template, replace_variable_in_template + + +def generate_cache_key(text: str, target_lang: str, source_lang: str = None, context: str = None) -> str: + cache_data = { + "text": text.strip(), + "target_lang": target_lang.lower(), + "source_lang": source_lang.lower() if source_lang else None, + "context": context.strip() if context else "" + } + + cache_string = json.dumps(cache_data, sort_keys=True, ensure_ascii=False) + return xxhash.xxh64(cache_string.encode('utf-8')).hexdigest() + +def translate_text(text: str, target_lang: str, source_lang: str = None, context: str = None) -> str: + if context: + prompt_text = get_template("translation_with_context") + prompt_text = replace_variable_in_template(prompt_text, "context", context) \ No newline at end of file diff --git a/config/config.py b/config/config.py index abd86a7..f85a6fc 100644 --- a/config/config.py +++ b/config/config.py @@ -148,7 +148,7 @@ class Config(object): }, } - SUPPORTED_LANGUAGES_Full = list(SUPPORTED_LANGUAGE_DETAILS.keys()) + SUPPORTED_LANGUAGES_FULL = list(SUPPORTED_LANGUAGE_DETAILS.keys()) # supported currencies SUPPORTED_CURRENCIES = ['€', '$'] @@ -156,10 +156,7 @@ class Config(object): # supported LLMs # SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'openai.text-embedding-3-large', 'mistral.mistral-embed'] SUPPORTED_EMBEDDINGS = ['mistral.mistral-embed'] - SUPPORTED_LLMS = ['openai.gpt-4o', 'openai.gpt-4o-mini', - 'mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest'] - - ANTHROPIC_LLM_VERSIONS = {'claude-3-5-sonnet': 'claude-3-5-sonnet-20240620', } + SUPPORTED_LLMS = ['mistral.mistral-large-latest', 'mistral.mistral-medium_latest', 'mistral.mistral-small-latest'] # Annotation text chunk length ANNOTATION_TEXT_CHUNK_LENGTH = 10000 diff --git a/config/prompts/globals/translation_with_context/1.0.0.yaml b/config/prompts/globals/translation_with_context/1.0.0.yaml new file mode 100644 index 0000000..3fd5d8d --- /dev/null +++ b/config/prompts/globals/translation_with_context/1.0.0.yaml @@ -0,0 +1,15 @@ +version: "1.0.0" +content: > + You are a top translator. We need you to translate {text_to_translate} into {target_language}, taking into account + this context: + + {context} + + I only want you to return the translation. No explanation, no options. I need to be able to directly use your answer + without further interpretation. If more than one option is available, present me with the most probable one. + +metadata: + author: "Josako" + date_added: "2025-06-23" + description: "An assistant to translate given a context." + changes: "Initial version" \ No newline at end of file diff --git a/config/prompts/globals/translation_without_context/1.0.0.yaml b/config/prompts/globals/translation_without_context/1.0.0.yaml new file mode 100644 index 0000000..1eece0b --- /dev/null +++ b/config/prompts/globals/translation_without_context/1.0.0.yaml @@ -0,0 +1,12 @@ +version: "1.0.0" +content: > + You are a top translator. We need you to translate {text_to_translate} into {target_language}. + + I only want you to return the translation. No explanation, no options. I need to be able to directly use your answer + without further interpretation. If more than one option is available, present me with the most probable one. + +metadata: + author: "Josako" + date_added: "2025-06-23" + description: "An assistant to translate without context." + changes: "Initial version" \ No newline at end of file diff --git a/config/type_defs/prompt_types.py b/config/type_defs/prompt_types.py index ec70635..769b3f2 100644 --- a/config/type_defs/prompt_types.py +++ b/config/type_defs/prompt_types.py @@ -28,4 +28,12 @@ PROMPT_TYPES = { "name": "transcript", "description": "An assistant to transform a transcript to markdown.", }, + "translation_with_context": { + "name": "translation_with_context", + "description": "An assistant to translate text with context", + }, + "translation_without_context": { + "name": "translation_without_context", + "description": "An assistant to translate text without context", + }, } diff --git a/eveai_app/views/dynamic_form_base.py b/eveai_app/views/dynamic_form_base.py index 7d41b31..f30dfc1 100644 --- a/eveai_app/views/dynamic_form_base.py +++ b/eveai_app/views/dynamic_form_base.py @@ -312,7 +312,7 @@ class DynamicFormBase(FlaskForm): field_class = SelectField tenant_id = session.get('tenant').get('id') makes = TenantMake.query.filter_by(tenant_id=tenant_id).all() - choices = [(make.name, make.name) for make in makes] + choices = [(make.id, make.name) for make in makes] extra_classes = '' field_kwargs = {'choices': choices} @@ -581,7 +581,10 @@ class DynamicFormBase(FlaskForm): except Exception as e: current_app.logger.error(f"Error converting initial data to patterns: {e}") elif isinstance(field, DateField): - data[original_field_name] = field.data.isoformat() + if field.data: + data[original_field_name] = field.data.isoformat() + else: + data[original_field_name] = None else: data[original_field_name] = field.data return data diff --git a/eveai_app/views/interaction_forms.py b/eveai_app/views/interaction_forms.py index d24a661..53e48bf 100644 --- a/eveai_app/views/interaction_forms.py +++ b/eveai_app/views/interaction_forms.py @@ -1,3 +1,4 @@ +from flask import session from flask_wtf import FlaskForm from wtforms import (StringField, BooleanField, SelectField, TextAreaField) from wtforms.fields.datetime import DateField @@ -181,7 +182,8 @@ class EditSpecialistMagicLinkForm(DynamicFormBase): self.specialist_name.data = '' # Dynamically populate the tenant_make field with None as first option - tenant_makes = TenantMake.query.all() + tenant_id = session.get('tenant').get('id') + tenant_makes = TenantMake.query.filter_by(tenant_id=tenant_id).all() self.tenant_make_id.choices = [(0, 'None')] + [(make.id, make.name) for make in tenant_makes] diff --git a/eveai_app/views/interaction_views.py b/eveai_app/views/interaction_views.py index 65fc846..94f6455 100644 --- a/eveai_app/views/interaction_views.py +++ b/eveai_app/views/interaction_views.py @@ -702,12 +702,13 @@ def specialist_magic_link(): new_spec_ml_tenant.tenant_id = tenant_id # Define the make valid for this magic link - make_id = SpecialistServices.get_specialist_system_field(new_specialist_magic_link.specialist_id, - "make", "tenant_make") + specialist = Specialist.query.get(new_specialist_magic_link.specialist_id) + make_id = specialist.configuration.get('make', None) + current_app.logger.debug(f"make_id defined in specialist: {make_id}") if make_id: - new_spec_ml_tenant.tenant_make_id = make_id + new_specialist_magic_link.tenant_make_id = make_id elif session.get('tenant').get('default_tenant_make_id'): - new_spec_ml_tenant.tenant_make_id = session.get('tenant').get('default_tenant_make_id') + new_specialist_magic_link.tenant_make_id = session.get('tenant').get('default_tenant_make_id') db.session.add(new_specialist_magic_link) db.session.add(new_spec_ml_tenant) diff --git a/eveai_app/views/partner_views.py b/eveai_app/views/partner_views.py index 38b8a0b..4befd26 100644 --- a/eveai_app/views/partner_views.py +++ b/eveai_app/views/partner_views.py @@ -62,6 +62,7 @@ def edit_partner(partner_id): update_logging_information(partner, dt.now(tz.utc)) db.session.commit() flash('Partner updated successfully.', 'success') + refresh_session_partner(partner.id) return redirect( prefixed_url_for('partner_bp.edit_partner', partner_id=partner.id)) # Assuming there's a user profile view to redirect to @@ -197,6 +198,7 @@ def edit_partner_service(partner_service_id): db.session.commit() flash('Partner Service updated successfully.', 'success') current_app.logger.info(f"Partner Service {partner_service.name} updated successfully! ") + refresh_session_partner(partner_id) except SQLAlchemyError as e: db.session.rollback() flash(f'Failed to update Partner Service: {str(e)}', 'danger') @@ -339,4 +341,7 @@ def add_partner_service_for_tenant(partner_service_id): return redirect(prefixed_url_for('partner_bp.partner_services')) - +def refresh_session_partner(partner_id): + if session.get('partner', None): + if partner_id == session['partner']['id']: + session['partner'] = Partner.query.get_or_404(partner_id).to_dict() diff --git a/eveai_chat_client/static/assets/js/chat-app.js b/eveai_chat_client/static/assets/js/chat-app.js index b7439de..f0413a3 100644 --- a/eveai_chat_client/static/assets/js/chat-app.js +++ b/eveai_chat_client/static/assets/js/chat-app.js @@ -131,18 +131,20 @@ export const ChatApp = { const historicalMessages = chatConfig.messages || []; if (historicalMessages.length > 0) { - this.allMessages = historicalMessages.map(msg => { - // Zorg voor een correct geformatteerde bericht-object - return { - id: this.messageIdCounter++, - content: typeof msg === 'string' ? msg : msg.content || '', - sender: msg.sender || 'ai', - type: msg.type || 'text', - timestamp: msg.timestamp || new Date().toISOString(), - formData: msg.formData || null, - status: msg.status || 'delivered' - }; - }); + this.allMessages = historicalMessages + .filter(msg => msg !== null && msg !== undefined) // Filter null/undefined berichten uit + .map(msg => { + // Zorg voor een correct geformatteerde bericht-object + return { + id: this.messageIdCounter++, + content: typeof msg === 'string' ? msg : (msg.content || ''), + sender: msg.sender || 'ai', + type: msg.type || 'text', + timestamp: msg.timestamp || new Date().toISOString(), + formData: msg.formData || null, + status: msg.status || 'delivered' + }; + }); console.log(`Loaded ${this.allMessages.length} historical messages`); } diff --git a/eveai_chat_client/views/chat_views.py b/eveai_chat_client/views/chat_views.py index 2c7ac25..dd4fb90 100644 --- a/eveai_chat_client/views/chat_views.py +++ b/eveai_chat_client/views/chat_views.py @@ -86,7 +86,13 @@ def chat(magic_link_code): session['chat_session_id'] = SpecialistServices.start_session() # Get customisation options with defaults - customisation = get_default_chat_customisation(tenant_make.chat_customisation_options) + current_app.logger.debug(f"Make Customisation Options: {tenant_make.chat_customisation_options}") + try: + customisation = get_default_chat_customisation(tenant_make.chat_customisation_options) + except Exception as e: + current_app.logger.error(f"Error processing customisation options: {str(e)}") + # Fallback to default customisation + customisation = get_default_chat_customisation(None) # Start a new chat session session['chat_session_id'] = SpecialistServices.start_session() diff --git a/eveai_chat_workers/specialists/traicie/TRAICIE_ROLE_DEFINITION_SPECIALIST/1_3.py b/eveai_chat_workers/specialists/traicie/TRAICIE_ROLE_DEFINITION_SPECIALIST/1_3.py index 0fcd666..3a973f3 100644 --- a/eveai_chat_workers/specialists/traicie/TRAICIE_ROLE_DEFINITION_SPECIALIST/1_3.py +++ b/eveai_chat_workers/specialists/traicie/TRAICIE_ROLE_DEFINITION_SPECIALIST/1_3.py @@ -69,6 +69,9 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor): self.role_definition_crew ) + def _config_state_result_relations(self): + pass + def execute(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult: self.log_tuning("Traicie Role Definition Specialist execution started", {}) diff --git a/requirements.txt b/requirements.txt index 47d356a..ebb187e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -94,4 +94,5 @@ scaleway~=2.9.0 html2text~=2025.4.15 markdown~=3.8 python-json-logger~=2.0.7 -qrcode[pil]==8.2 \ No newline at end of file +qrcode[pil]==8.2 +xxhash~=3.5.0 \ No newline at end of file