From 503ea7965da10e90cb46c91f8f2ddf2ee1d04fd9 Mon Sep 17 00:00:00 2001 From: Josako Date: Sun, 3 Nov 2024 16:18:14 +0100 Subject: [PATCH] - Temporary checkin to branch for the rest of the introduction of experts --- .repopackignore_eveai_api | 2 -- CHANGELOG.md | 25 +++++++++++++++++++++++++ common/models/document.py | 1 + common/models/interaction.py | 17 ++++++++++++++++- config/catalog_types.py | 2 +- config/retriever_types.py | 2 +- config/specialist_types.py | 23 +++++++++++++++++++++++ eveai_app/views/dynamic_form_base.py | 1 + eveai_chat_workers/tasks.py | 4 ++-- 9 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 config/specialist_types.py diff --git a/.repopackignore_eveai_api b/.repopackignore_eveai_api index 900294e..b238181 100644 --- a/.repopackignore_eveai_api +++ b/.repopackignore_eveai_api @@ -4,8 +4,6 @@ eveai_beat/ eveai_chat/ eveai_chat_workers/ eveai_entitlements/ -eveai_workers/ instance/ -integrations/ nginx/ scripts/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eb9ff3..a48e9f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - In case of vulnerabilities. +## [1.0.14-alfa] + +### Added +- New release script added to tag images with release number +- Allow the addition of multiple types of Catalogs +- Generic functionality to enable dynamic fields +- Addition of Retrievers to allow for smart collection of information in Catalogs +- Add dynamic fields to Catalog / Retriever / DocumentVersion + +### Changed +- Processing parameters defined at Catalog level iso Tenant level +- Reroute 'blank' paths to 'admin' + +### Deprecated +- For soon-to-be removed features. + +### Removed +- For now removed features. + +### Fixed +- Set default language when registering Documents or URLs. + +### Security +- In case of vulnerabilities. + ## [1.0.13-alfa] ### Added diff --git a/common/models/document.py b/common/models/document.py index 4f383f9..a904f46 100644 --- a/common/models/document.py +++ b/common/models/document.py @@ -53,6 +53,7 @@ class Retriever(db.Model): user_metadata = db.Column(JSONB, nullable=True) system_metadata = db.Column(JSONB, nullable=True) configuration = db.Column(JSONB, nullable=True) + arguments = db.Column(JSONB, nullable=True) # Versioning Information created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.now()) diff --git a/common/models/interaction.py b/common/models/interaction.py index 4758aa3..450eb96 100644 --- a/common/models/interaction.py +++ b/common/models/interaction.py @@ -1,6 +1,8 @@ +from sqlalchemy.dialects.postgresql import JSONB + from ..extensions import db from .user import User, Tenant -from .document import Embedding +from .document import Embedding, Retriever class ChatSession(db.Model): @@ -44,3 +46,16 @@ class Interaction(db.Model): class InteractionEmbedding(db.Model): interaction_id = db.Column(db.Integer, db.ForeignKey(Interaction.id, ondelete='CASCADE'), primary_key=True) embedding_id = db.Column(db.Integer, db.ForeignKey(Embedding.id, ondelete='CASCADE'), primary_key=True) + + +class Specialist(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(20), nullable=False) + tuning = db.Column(db.Boolean, nullable=True, default=False) + configuration = db.Column(JSONB, nullable=True) + arguments = db.Column(JSONB, nullable=True) + + +class SpecialistRetriever(db.Model): + specialist_id = db.Column(db.Integer, db.ForeignKey(Specialist.id, ondelete='CASCADE'), primary_key=True) + retriever_id = db.Column(db.Integer, db.ForeignKey(Retriever.id, ondelete='CASCADE'), primary_key=True) diff --git a/config/catalog_types.py b/config/catalog_types.py index 756cd9a..a5c2cc5 100644 --- a/config/catalog_types.py +++ b/config/catalog_types.py @@ -1,6 +1,6 @@ # Catalog Types CATALOG_TYPES = { - "DEFAULT": { + "STANDARD": { "name": "Default Catalog", "Description": "A Catalog with information in Evie's Library, to be considered as a whole", "configuration": {} diff --git a/config/retriever_types.py b/config/retriever_types.py index 2a4f1c5..92673da 100644 --- a/config/retriever_types.py +++ b/config/retriever_types.py @@ -1,6 +1,6 @@ # Retriever Types RETRIEVER_TYPES = { - "DEFAULT_RAG": { + "STANDARD_RAG": { "name": "Default RAG", "description": "Retrieving all embeddings conform the query", "configuration": { diff --git a/config/specialist_types.py b/config/specialist_types.py new file mode 100644 index 0000000..14b1d7c --- /dev/null +++ b/config/specialist_types.py @@ -0,0 +1,23 @@ +# Specialist Types +SPECIALIST_TYPES = { + "STANDARD_RAG": { + "name": "Q&A RAG Specialist", + "description": "Standard Q&A through RAG Specialist", + "configuration": { + "specialist_context": { + "name": "Specialist Context", + "type": "text", + "description": "The context to be used by the specialist.", + "required": False, + }, + }, + "arguments": { + "language": { + "name": "Language", + "type": "str", + "description": "Language code to be used for receiving questions and giving answers", + "required": True, + }, + } + } +} diff --git a/eveai_app/views/dynamic_form_base.py b/eveai_app/views/dynamic_form_base.py index 0abe4e9..f518003 100644 --- a/eveai_app/views/dynamic_form_base.py +++ b/eveai_app/views/dynamic_form_base.py @@ -75,6 +75,7 @@ class DynamicFormBase(FlaskForm): 'float': FloatField, 'boolean': BooleanField, 'string': StringField, + 'text': TextAreaField, 'date': DateField, }.get(field_type, StringField) field_kwargs = {} diff --git a/eveai_chat_workers/tasks.py b/eveai_chat_workers/tasks.py index 719a1cd..6e124ed 100644 --- a/eveai_chat_workers/tasks.py +++ b/eveai_chat_workers/tasks.py @@ -132,7 +132,7 @@ def answer_using_tenant_rag(question, language, tenant, chat_session): new_interaction.detailed_question_at = dt.now(tz.utc) with current_event.create_span("Generate Answer using RAG"): - retriever = EveAIDefaultRagRetriever(model_variables, tenant_info) + retriever = EveAIDefaultRagRetriever(1, model_variables, tenant_info) llm = model_variables['llm'] template = model_variables['rag_template'] language_template = create_language_template(template, language) @@ -236,7 +236,7 @@ def answer_using_llm(question, language, tenant, chat_session): new_interaction.detailed_question_at = dt.now(tz.utc) with current_event.create_span("Detail Answer using LLM"): - retriever = EveAIDefaultRagRetriever(model_variables, tenant_info) + retriever = EveAIDefaultRagRetriever(1, model_variables, tenant_info) llm = model_variables['llm_no_rag'] template = model_variables['encyclopedia_template'] language_template = create_language_template(template, language)