diff --git a/common/models/user.py b/common/models/user.py index aa0ad27..d69c440 100644 --- a/common/models/user.py +++ b/common/models/user.py @@ -341,3 +341,16 @@ class TranslationCache(db.Model): updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True) last_used_at = db.Column(db.DateTime, nullable=True) + + +class PartnerRAGRetriever(db.Model): + __bind_key__ = 'public' + __table_args__ = ( + {'schema': 'public'}, + db.PrimaryKeyConstraint('tenant_id', 'retriever_id'), + db.UniqueConstraint('partner_id', 'tenant_id', 'retriever_id') + ) + + partner_id = db.Column(db.Integer, db.ForeignKey('public.partner.id'), nullable=False) + tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False) + retriever_id = db.Column(db.Integer, nullable=False) diff --git a/common/services/entitlements/license_period_services.py b/common/services/entitlements/license_period_services.py index 569fde1..a9b9b86 100644 --- a/common/services/entitlements/license_period_services.py +++ b/common/services/entitlements/license_period_services.py @@ -41,7 +41,7 @@ class LicensePeriodServices: current_app.logger.debug(f"Found license period {license_period.id} for tenant {tenant_id} " f"with status {license_period.status}") match license_period.status: - case PeriodStatus.UPCOMING: + case PeriodStatus.UPCOMING | PeriodStatus.PENDING: current_app.logger.debug(f"In upcoming state") LicensePeriodServices._complete_last_license_period(tenant_id=tenant_id) current_app.logger.debug(f"Completed last license period for tenant {tenant_id}") @@ -73,8 +73,6 @@ class LicensePeriodServices: raise EveAIPendingLicensePeriod() case PeriodStatus.ACTIVE: return license_period - case PeriodStatus.PENDING: - return license_period else: raise EveAILicensePeriodsExceeded(license_id=None) except SQLAlchemyError as e: @@ -125,7 +123,7 @@ class LicensePeriodServices: tenant_id=tenant_id, period_number=next_period_number, period_start=the_license.start_date + relativedelta(months=next_period_number-1), - period_end=the_license.end_date + relativedelta(months=next_period_number, days=-1), + period_end=the_license.start_date + relativedelta(months=next_period_number, days=-1), status=PeriodStatus.UPCOMING, upcoming_at=dt.now(tz.utc), ) diff --git a/config/agents/evie_partner/PARTNER_RAG_AGENT/1.0.0.yaml b/config/agents/evie_partner/PARTNER_RAG_AGENT/1.0.0.yaml new file mode 100644 index 0000000..b8ea576 --- /dev/null +++ b/config/agents/evie_partner/PARTNER_RAG_AGENT/1.0.0.yaml @@ -0,0 +1,26 @@ +version: "1.0.0" +name: "Partner Rag Agent" +role: > + You are a virtual assistant responsible for answering user questions about the Evie platform (Ask Eve AI) and products + developed by partners on top of it. You are reliable point of contact for end-users seeking help, clarification, or + deeper understanding of features, capabilities, integrations, or workflows related to these AI-powered solutions. +goal: > + Your primary goal is to: + • Provide clear, relevant, and accurate responses to user questions. + • Reduce friction in user onboarding and daily usage. + • Increase user confidence and adoption of both the platform and partner-developed products. + • Act as a bridge between documentation and practical application, enabling users to help themselves through intelligent guidance. +backstory: > + You have availability Evie’s own documentation, partner product manuals, and real user interactions. You are designed + to replace passive documentation with active, contextual assistance. + You have evolved beyond a support bot: you combine knowledge, reasoning, and a friendly tone to act as a product + companion that grows with the ecosystem. As partner products expand, the agent updates its knowledge and learns to + distinguish between general platform capabilities and product-specific nuances, offering a personalised experience + each time. +full_model_name: "mistral.mistral-medium-latest" +temperature: 0.3 +metadata: + author: "Josako" + date_added: "2025-07-16" + description: "An Agent that does RAG based on a user's question, RAG content & history" + changes: "Initial version" diff --git a/config/partner_services/globals/KNOWLEDGE_SERVICE/1.0.0.yaml b/config/partner_services/globals/KNOWLEDGE_SERVICE/1.0.0.yaml new file mode 100644 index 0000000..2e63a0b --- /dev/null +++ b/config/partner_services/globals/KNOWLEDGE_SERVICE/1.0.0.yaml @@ -0,0 +1,9 @@ +version: "1.0.0" +name: "Knowledge Service" +configuration: {} +permissions: {} +metadata: + author: "Josako" + date_added: "2025-04-02" + changes: "Initial version" + description: "Partner providing catalog content" diff --git a/config/retrievers/evie_partner/PARTNER_RAG/1.0.0.yaml b/config/retrievers/evie_partner/PARTNER_RAG/1.0.0.yaml new file mode 100644 index 0000000..6038cbb --- /dev/null +++ b/config/retrievers/evie_partner/PARTNER_RAG/1.0.0.yaml @@ -0,0 +1,21 @@ +version: "1.0.0" +name: "Standard RAG Retriever" +configuration: + es_k: + name: "es_k" + type: "integer" + description: "K-value to retrieve embeddings (max embeddings retrieved)" + required: true + default: 8 + es_similarity_threshold: + name: "es_similarity_threshold" + type: "float" + description: "Similarity threshold for retrieving embeddings" + required: true + default: 0.3 +arguments: {} +metadata: + author: "Josako" + date_added: "2025-01-24" + changes: "Initial version" + description: "Retrieving all embeddings conform the query" diff --git a/config/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1.0.0.yaml b/config/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1.0.0.yaml new file mode 100644 index 0000000..c27c5a4 --- /dev/null +++ b/config/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1.0.0.yaml @@ -0,0 +1,34 @@ +version: "1.0.0" +name: "Partner RAG Specialist" +framework: "crewai" +chat: true +configuration: {} +arguments: {} +results: + rag_output: + answer: + name: "answer" + type: "str" + description: "Answer to the query" + required: true + citations: + name: "citations" + type: "List[str]" + description: "List of citations" + required: false + insufficient_info: + name: "insufficient_info" + type: "bool" + description: "Whether or not the query is insufficient info" + required: true +agents: + - type: "PARTNER_RAG_AGENT" + version: "1.0" +tasks: + - type: "PARTNER_RAG_TASK" + version: "1.0" +metadata: + author: "Josako" + date_added: "2025-07-16" + changes: "Initial version" + description: "Q&A through Partner RAG Specialist (for documentation purposes)" \ No newline at end of file diff --git a/config/specialists/globals/RAG_SPECIALIST/1.1.0.yaml b/config/specialists/globals/RAG_SPECIALIST/1.1.0.yaml index d6f73de..70b2b10 100644 --- a/config/specialists/globals/RAG_SPECIALIST/1.1.0.yaml +++ b/config/specialists/globals/RAG_SPECIALIST/1.1.0.yaml @@ -1,4 +1,4 @@ -version: "1.0.0" +version: "1.1.0" name: "RAG Specialist" framework: "crewai" chat: true diff --git a/config/specialists/globals/STANDARD_RAG_SPECIALIST/1.0.0.yaml b/config/specialists/globals/STANDARD_RAG_SPECIALIST/1.0.0.yaml deleted file mode 100644 index 5c5c1c9..0000000 --- a/config/specialists/globals/STANDARD_RAG_SPECIALIST/1.0.0.yaml +++ /dev/null @@ -1,53 +0,0 @@ -version: 1.0.0 -name: "Standard RAG Specialist" -framework: "langchain" -chat: true -configuration: - specialist_context: - name: "Specialist Context" - type: "text" - description: "The context to be used by the specialist." - required: false - temperature: - name: "Temperature" - type: "number" - description: "The inference temperature to be used by the specialist." - required: false - default: 0.3 -arguments: - language: - name: "Language" - type: "str" - description: "Language code to be used for receiving questions and giving answers" - required: true - query: - name: "query" - type: "str" - description: "Query to answer" - required: true -results: - detailed_query: - name: "detailed_query" - type: "str" - description: "The query detailed with the Chat Session History." - required: true - answer: - name: "answer" - type: "str" - description: "Answer to the query" - required: true - citations: - name: "citations" - type: "List[str]" - description: "List of citations" - required: false - insufficient_info: - name: "insufficient_info" - type: "bool" - description: "Whether or not the query is insufficient info" - required: true -metadata: - author: "Josako" - date_added: "2025-01-08" - changes: "Initial version" - description: "A Specialist that performs standard Q&A" \ No newline at end of file diff --git a/config/tasks/evie_partner/PARTNER_RAG_TASK/1.0.0.yaml b/config/tasks/evie_partner/PARTNER_RAG_TASK/1.0.0.yaml new file mode 100644 index 0000000..7ecdde6 --- /dev/null +++ b/config/tasks/evie_partner/PARTNER_RAG_TASK/1.0.0.yaml @@ -0,0 +1,22 @@ +version: "1.0.0" +name: "RAG Task" +task_description: > + Answer the question based on the following context, and taking into account the history of the discussion. Try not to + repeat answers already given in the recent history, unless confirmation is required or repetition is essential to + give a coherent answer. + Answer the end user in the language used in his/her question. + If the question cannot be answered using the given context, answer "I have insufficient information to answer this + question." + Context (in between triple $): + $$${context}$$$ + History (in between triple €): + €€€{history}€€€ + Question (in between triple £): + £££{question}£££ +expected_output: > + Your answer. +metadata: + author: "Josako" + date_added: "2025-07-16" + description: "A Task that gives RAG-based answers" + changes: "Initial version" diff --git a/config/type_defs/partner_service_types.py b/config/type_defs/partner_service_types.py index a6fa34d..e849d42 100644 --- a/config/type_defs/partner_service_types.py +++ b/config/type_defs/partner_service_types.py @@ -1,9 +1,5 @@ # config/type_defs/partner_service_types.py PARTNER_SERVICE_TYPES = { - "REFERRAL_SERVICE": { - "name": "Referral Service", - "description": "Partner referring new customers", - }, "KNOWLEDGE_SERVICE": { "name": "Knowledge Service", "description": "Partner providing catalog content", diff --git a/config/type_defs/retriever_types.py b/config/type_defs/retriever_types.py index eca5e03..970bbbd 100644 --- a/config/type_defs/retriever_types.py +++ b/config/type_defs/retriever_types.py @@ -4,6 +4,11 @@ RETRIEVER_TYPES = { "name": "Standard RAG Retriever", "description": "Retrieving all embeddings from the catalog conform the query", }, + "PARTNER_RAG": { + "name": "Partner RAG Retriever", + "description": "RAG intended for partner documentation", + "partner": "evie_partner" + }, "TRAICIE_ROLE_DEFINITION_BY_ROLE_IDENTIFICATION": { "name": "Traicie Role Definition Retriever by Role Identification", "description": "Retrieves relevant role information for a given role", diff --git a/config/type_defs/specialist_types.py b/config/type_defs/specialist_types.py index a51709d..a7274f0 100644 --- a/config/type_defs/specialist_types.py +++ b/config/type_defs/specialist_types.py @@ -1,13 +1,14 @@ # Specialist Types SPECIALIST_TYPES = { - "STANDARD_RAG_SPECIALIST": { - "name": "Standard RAG Specialist", - "description": "Standard Q&A through RAG Specialist", - }, "RAG_SPECIALIST": { "name": "RAG Specialist", "description": "Q&A through RAG Specialist", }, + "PARTNER_RAG_SPECIALIST": { + "name": "Partner RAG Specialist", + "description": "Q&A through Partner RAG Specialist (for documentation purposes)", + "partner": "evie_partner" + }, "SPIN_SPECIALIST": { "name": "Spin Sales Specialist", "description": "A specialist that allows to answer user queries, try to get SPIN-information and Identification", diff --git a/documentation/document_domain.mermaid b/documentation/document_domain.mermaid new file mode 100644 index 0000000..0221efb --- /dev/null +++ b/documentation/document_domain.mermaid @@ -0,0 +1,138 @@ +erDiagram + CATALOG { + int id PK + string name + text description + string type + string type_version + int min_chunk_size + int max_chunk_size + jsonb user_metadata + jsonb system_metadata + jsonb configuration + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PROCESSOR { + int id PK + string name + text description + int catalog_id FK + string type + string sub_file_type + boolean active + boolean tuning + jsonb user_metadata + jsonb system_metadata + jsonb configuration + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + RETRIEVER { + int id PK + string name + text description + int catalog_id FK + string type + string type_version + boolean tuning + jsonb user_metadata + jsonb system_metadata + jsonb configuration + jsonb arguments + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + DOCUMENT { + int id PK + int catalog_id FK + string name + datetime valid_from + datetime valid_to + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + DOCUMENT_VERSION { + int id PK + int doc_id FK + string url + string bucket_name + string object_name + string file_type + string sub_file_type + float file_size + string language + text user_context + text system_context + jsonb user_metadata + jsonb system_metadata + jsonb catalog_properties + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + boolean processing + datetime processing_started_at + datetime processing_finished_at + string processing_error + } + + EMBEDDING { + int id PK + string type + int doc_vers_id FK + boolean active + text chunk + } + + EMBEDDING_MISTRAL { + int id PK,FK + vector_1024 embedding + } + + EMBEDDING_SMALL_OPENAI { + int id PK,FK + vector_1536 embedding + } + + EMBEDDING_LARGE_OPENAI { + int id PK,FK + vector_3072 embedding + } + + USER { + int id PK + string user_name + string email + } + + %% Relationships + CATALOG ||--o{ PROCESSOR : "has many" + CATALOG ||--o{ RETRIEVER : "has many" + CATALOG ||--o{ DOCUMENT : "has many" + + DOCUMENT ||--o{ DOCUMENT_VERSION : "has many" + + DOCUMENT_VERSION ||--o{ EMBEDDING : "has many" + + EMBEDDING ||--o| EMBEDDING_MISTRAL : "inheritance" + EMBEDDING ||--o| EMBEDDING_SMALL_OPENAI : "inheritance" + EMBEDDING ||--o| EMBEDDING_LARGE_OPENAI : "inheritance" + + USER ||--o{ CATALOG : "creates/updates" + USER ||--o{ PROCESSOR : "creates/updates" + USER ||--o{ RETRIEVER : "creates/updates" + USER ||--o{ DOCUMENT : "creates/updates" + USER ||--o{ DOCUMENT_VERSION : "creates/updates" \ No newline at end of file diff --git a/documentation/entitlements_domain.mermaid b/documentation/entitlements_domain.mermaid new file mode 100644 index 0000000..cc39481 --- /dev/null +++ b/documentation/entitlements_domain.mermaid @@ -0,0 +1,244 @@ +erDiagram + BUSINESS_EVENT_LOG { + int id PK + datetime timestamp + string event_type + int tenant_id + string trace_id + string span_id + string span_name + string parent_span_id + int document_version_id + float document_version_file_size + int specialist_id + string specialist_type + string specialist_type_version + string chat_session_id + int interaction_id + string environment + int llm_metrics_total_tokens + int llm_metrics_prompt_tokens + int llm_metrics_completion_tokens + float llm_metrics_total_time + int llm_metrics_nr_of_pages + int llm_metrics_call_count + string llm_interaction_type + text message + int license_usage_id FK + } + + LICENSE { + int id PK + int tenant_id FK + int tier_id FK + date start_date + date end_date + int nr_of_periods + string currency + boolean yearly_payment + float basic_fee + int max_storage_mb + float additional_storage_price + int additional_storage_bucket + int included_embedding_mb + decimal additional_embedding_price + int additional_embedding_bucket + int included_interaction_tokens + decimal additional_interaction_token_price + int additional_interaction_bucket + float overage_embedding + float overage_interaction + boolean additional_storage_allowed + boolean additional_embedding_allowed + boolean additional_interaction_allowed + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + LICENSE_TIER { + int id PK + string name + string version + date start_date + date end_date + float basic_fee_d + float basic_fee_e + int max_storage_mb + decimal additional_storage_price_d + decimal additional_storage_price_e + int additional_storage_bucket + int included_embedding_mb + decimal additional_embedding_price_d + decimal additional_embedding_price_e + int additional_embedding_bucket + int included_interaction_tokens + decimal additional_interaction_token_price_d + decimal additional_interaction_token_price_e + int additional_interaction_bucket + float standard_overage_embedding + float standard_overage_interaction + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PARTNER_SERVICE_LICENSE_TIER { + int partner_service_id PK,FK + int license_tier_id PK,FK + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + LICENSE_PERIOD { + int id PK + int license_id FK + int tenant_id FK + int period_number + date period_start + date period_end + string currency + float basic_fee + int max_storage_mb + float additional_storage_price + int additional_storage_bucket + int included_embedding_mb + decimal additional_embedding_price + int additional_embedding_bucket + int included_interaction_tokens + decimal additional_interaction_token_price + int additional_interaction_bucket + boolean additional_storage_allowed + boolean additional_embedding_allowed + boolean additional_interaction_allowed + enum status + datetime upcoming_at + datetime pending_at + datetime active_at + datetime completed_at + datetime invoiced_at + datetime closed_at + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + LICENSE_USAGE { + int id PK + int tenant_id FK + float storage_mb_used + float embedding_mb_used + int embedding_prompt_tokens_used + int embedding_completion_tokens_used + int embedding_total_tokens_used + int interaction_prompt_tokens_used + int interaction_completion_tokens_used + int interaction_total_tokens_used + int license_period_id FK + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PAYMENT { + int id PK + int license_period_id FK + int tenant_id FK + enum payment_type + decimal amount + string currency + text description + enum status + string external_payment_id + string payment_method + jsonb provider_data + datetime paid_at + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + INVOICE { + int id PK + int license_period_id FK + int payment_id FK + int tenant_id FK + enum invoice_type + string invoice_number + date invoice_date + date due_date + decimal amount + string currency + decimal tax_amount + text description + enum status + datetime sent_at + datetime paid_at + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + LICENSE_CHANGE_LOG { + int id PK + int license_id FK + datetime changed_at + string field_name + string old_value + string new_value + text reason + int created_by FK + } + + TENANT { + int id PK + string name + string currency + } + + USER { + int id PK + string user_name + string email + } + + PARTNER_SERVICE { + int id PK + string name + string type + } + + %% Main business relationships + TENANT ||--o{ LICENSE : "has many" + LICENSE_TIER ||--o{ LICENSE : "has many" + LICENSE ||--o{ LICENSE_PERIOD : "has many" + LICENSE_PERIOD ||--|| LICENSE_USAGE : "has one" + LICENSE_PERIOD ||--o{ PAYMENT : "has many" + LICENSE_PERIOD ||--o{ INVOICE : "has many" + + %% License management + LICENSE ||--o{ LICENSE_CHANGE_LOG : "has many" + + %% Payment-Invoice relationship + PAYMENT ||--o| INVOICE : "can have" + + %% Partner service licensing + PARTNER_SERVICE ||--o{ PARTNER_SERVICE_LICENSE_TIER : "has many" + LICENSE_TIER ||--o{ PARTNER_SERVICE_LICENSE_TIER : "has many" + + %% Event logging + LICENSE_USAGE ||--o{ BUSINESS_EVENT_LOG : "has many" + + %% Tenant relationships + TENANT ||--o{ LICENSE_PERIOD : "has many" + TENANT ||--o{ LICENSE_USAGE : "has many" + TENANT ||--o{ PAYMENT : "has many" + TENANT ||--o{ INVOICE : "has many" \ No newline at end of file diff --git a/documentation/interaction_domain.mermaid b/documentation/interaction_domain.mermaid new file mode 100644 index 0000000..138dcc2 --- /dev/null +++ b/documentation/interaction_domain.mermaid @@ -0,0 +1,211 @@ +erDiagram + CHAT_SESSION { + int id PK + int user_id FK + string session_id + datetime session_start + datetime session_end + string timezone + } + + SPECIALIST { + int id PK + string name + text description + string type + string type_version + boolean tuning + jsonb configuration + jsonb arguments + boolean active + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + EVE_AI_ASSET { + int id PK + string name + text description + string type + string type_version + string bucket_name + string object_name + string file_type + float file_size + jsonb user_metadata + jsonb system_metadata + jsonb configuration + int prompt_tokens + int completion_tokens + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + datetime last_used_at + } + + EVE_AI_AGENT { + int id PK + int specialist_id FK + string name + text description + string type + string type_version + text role + text goal + text backstory + boolean tuning + jsonb configuration + jsonb arguments + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + EVE_AI_TASK { + int id PK + int specialist_id FK + string name + text description + string type + string type_version + text task_description + text expected_output + boolean tuning + jsonb configuration + jsonb arguments + jsonb context + boolean asynchronous + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + EVE_AI_TOOL { + int id PK + int specialist_id FK + string name + text description + string type + string type_version + boolean tuning + jsonb configuration + jsonb arguments + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + DISPATCHER { + int id PK + string name + text description + string type + string type_version + boolean tuning + jsonb configuration + jsonb arguments + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + INTERACTION { + int id PK + int chat_session_id FK + int specialist_id FK + jsonb specialist_arguments + jsonb specialist_results + string timezone + int appreciation + datetime question_at + datetime detailed_question_at + datetime answer_at + string processing_error + } + + INTERACTION_EMBEDDING { + int interaction_id PK,FK + int embedding_id PK,FK + } + + SPECIALIST_RETRIEVER { + int specialist_id PK,FK + int retriever_id PK,FK + } + + SPECIALIST_DISPATCHER { + int specialist_id PK,FK + int dispatcher_id PK,FK + } + + SPECIALIST_MAGIC_LINK { + int id PK + string name + text description + int specialist_id FK + int tenant_make_id FK + string magic_link_code + datetime valid_from + datetime valid_to + jsonb specialist_args + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + USER { + int id PK + string user_name + string email + } + + TENANT_MAKE { + int id PK + string name + text description + } + + RETRIEVER { + int id PK + string name + text description + } + + EMBEDDING { + int id PK + string type + text chunk + } + + %% Main conversation flow + USER ||--o{ CHAT_SESSION : "has many" + CHAT_SESSION ||--o{ INTERACTION : "has many" + SPECIALIST ||--o{ INTERACTION : "processes" + + %% Specialist composition (EveAI components) + SPECIALIST ||--o{ EVE_AI_AGENT : "has many" + SPECIALIST ||--o{ EVE_AI_TASK : "has many" + SPECIALIST ||--o{ EVE_AI_TOOL : "has many" + + %% Specialist connections + SPECIALIST ||--o{ SPECIALIST_RETRIEVER : "uses retrievers" + RETRIEVER ||--o{ SPECIALIST_RETRIEVER : "used by specialists" + + SPECIALIST ||--o{ SPECIALIST_DISPATCHER : "uses dispatchers" + DISPATCHER ||--o{ SPECIALIST_DISPATCHER : "used by specialists" + + %% Interaction results + INTERACTION ||--o{ INTERACTION_EMBEDDING : "references embeddings" + EMBEDDING ||--o{ INTERACTION_EMBEDDING : "used in interactions" + + %% Magic links for specialist access + SPECIALIST ||--o{ SPECIALIST_MAGIC_LINK : "has magic links" + TENANT_MAKE ||--o{ SPECIALIST_MAGIC_LINK : "branded links" \ No newline at end of file diff --git a/documentation/user_domain.mermaid b/documentation/user_domain.mermaid new file mode 100644 index 0000000..386606c --- /dev/null +++ b/documentation/user_domain.mermaid @@ -0,0 +1,199 @@ +erDiagram + TENANT { + int id PK + string code + string name + string website + string timezone + string type + string currency + boolean storage_dirty + int default_tenant_make_id FK + datetime created_at + datetime updated_at + } + + USER { + int id PK + int tenant_id FK + string user_name + string email + string password + string first_name + string last_name + boolean active + string fs_uniquifier + datetime confirmed_at + date valid_to + boolean is_primary_contact + boolean is_financial_contact + datetime last_login_at + datetime current_login_at + string last_login_ip + string current_login_ip + int login_count + datetime created_at + datetime updated_at + } + + ROLE { + int id PK + string name + string description + } + + ROLES_USERS { + int user_id PK,FK + int role_id PK,FK + } + + TENANT_DOMAIN { + int id PK + int tenant_id FK + string domain + date valid_to + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + TENANT_PROJECT { + int id PK + int tenant_id FK + string name + text description + array services + string encrypted_api_key + string visual_api_key + boolean active + string responsible_email + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + TENANT_MAKE { + int id PK + int tenant_id FK + string name + text description + boolean active + string website + string logo_url + string default_language + array allowed_languages + jsonb chat_customisation_options + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PARTNER { + int id PK + int tenant_id FK + string code + string logo_url + boolean active + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PARTNER_SERVICE { + int id PK + int partner_id FK + string name + text description + string type + string type_version + boolean active + json configuration + json permissions + json system_metadata + json user_metadata + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PARTNER_TENANT { + int partner_service_id PK,FK + int tenant_id PK,FK + json configuration + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + } + + PARTNER_SERVICE_LICENSE_TIER { + int id PK + int partner_service_id FK + } + + SPECIALIST_MAGIC_LINK_TENANT { + string magic_link_code PK + int tenant_id FK + } + + TRANSLATION_CACHE { + string cache_key PK + text source_text + text translated_text + string source_language + string target_language + text context + int prompt_tokens + int completion_tokens + datetime created_at + int created_by FK + datetime updated_at + int updated_by FK + datetime last_used_at + } + + PARTNER_RAG_RETRIEVER { + int partner_id FK + int tenant_id PK,FK + int retriever_id PK + } + + LICENSE { + int id PK + int tenant_id FK + date start_date + date end_date + } + + LICENSE_USAGE { + int id PK + int tenant_id FK + } + + %% Relationships + TENANT ||--o{ USER : "has many" + TENANT ||--o{ TENANT_DOMAIN : "has many" + TENANT ||--o{ TENANT_PROJECT : "has many" + TENANT ||--o{ TENANT_MAKE : "has many" + TENANT ||--o| TENANT_MAKE : "default_tenant_make" + TENANT ||--o| PARTNER : "has one" + TENANT ||--o{ LICENSE : "has many" + TENANT ||--o{ LICENSE_USAGE : "has many" + TENANT ||--o{ SPECIALIST_MAGIC_LINK_TENANT : "has many" + TENANT ||--o{ PARTNER_TENANT : "has many" + TENANT ||--o{ PARTNER_RAG_RETRIEVER : "has many" + + USER ||--o{ ROLES_USERS : "has many" + + ROLE ||--o{ ROLES_USERS : "has many" + + PARTNER ||--o{ PARTNER_SERVICE : "has many" + PARTNER ||--o{ PARTNER_RAG_RETRIEVER : "has many" + + PARTNER_SERVICE ||--o{ PARTNER_TENANT : "has many" + PARTNER_SERVICE ||--o{ PARTNER_SERVICE_LICENSE_TIER : "has many" \ No newline at end of file diff --git a/eveai_app/templates/entitlements/edit_license_period.html b/eveai_app/templates/entitlements/edit_license_period.html new file mode 100644 index 0000000..1eb3308 --- /dev/null +++ b/eveai_app/templates/entitlements/edit_license_period.html @@ -0,0 +1,24 @@ +{% extends 'base.html' %} +{% from "macros.html" import render_field %} + +{% block title %}Edit Tenant Project{% endblock %} + +{% block content_title %}Edit License Period{% endblock %} +{% block content_description %}Edit a License Period{% endblock %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {% set disabled_fields = [] %} + {% set exclude_fields = [] %} + + {% for field in form %} + {{ render_field(field, disabled_fields, exclude_fields) }} + {% endfor %} + +
+{% endblock %} + +{% block content_footer %} + +{% endblock %} diff --git a/eveai_app/templates/navbar.html b/eveai_app/templates/navbar.html index 2c4cd37..5fc4c46 100644 --- a/eveai_app/templates/navbar.html +++ b/eveai_app/templates/navbar.html @@ -72,6 +72,7 @@ {'name': 'Tenants', 'url': '/user/tenants', 'roles': ['Super User', 'Partner Admin']}, {'name': 'Tenant Overview', 'url': '/user/tenant_overview', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Edit Tenant', 'url': '/user/tenant/' ~ session['tenant'].get('id'), 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, + {'name': 'Tenant Partner Services', 'url': '/user/tenant_partner_services', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Tenant Makes', 'url': '/user/tenant_makes', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Tenant Projects', 'url': '/user/tenant_projects', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, {'name': 'Users', 'url': '/user/view_users', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']}, diff --git a/eveai_app/views/document_views.py b/eveai_app/views/document_views.py index 9677f93..50719d6 100644 --- a/eveai_app/views/document_views.py +++ b/eveai_app/views/document_views.py @@ -455,10 +455,10 @@ def add_url(): except EveAIException as e: current_app.logger.error(f"Error adding document: {str(e)}") - flash(str(e), 'error') + flash(str(e), 'danger') except Exception as e: current_app.logger.error(f'Error adding document: {str(e)}') - flash('An error occurred while adding the document.', 'error') + flash('An error occurred while adding the document.', 'danger') return render_template('document/add_url.html', form=form) diff --git a/eveai_app/views/entitlements_forms.py b/eveai_app/views/entitlements_forms.py index 325a7e9..7b80f19 100644 --- a/eveai_app/views/entitlements_forms.py +++ b/eveai_app/views/entitlements_forms.py @@ -5,6 +5,8 @@ from wtforms import (StringField, PasswordField, BooleanField, SubmitField, Emai from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError, InputRequired import pytz +from common.models.entitlements import PeriodStatus + class LicenseTierForm(FlaskForm): name = StringField('Name', validators=[DataRequired(), Length(max=50)]) @@ -76,3 +78,14 @@ class LicenseForm(FlaskForm): validators=[DataRequired(), NumberRange(min=0)], default=0) + +class EditPeriodForm(FlaskForm): + period_start = DateField('Period Start', id='form-control datepicker', validators=[DataRequired()]) + period_end = DateField('Period End', id='form-control datepicker', validators=[DataRequired()]) + status = SelectField('Status', choices=[], validators=[DataRequired()]) + + def __init__(self, *args, **kwargs): + super(EditPeriodForm, self).__init__(*args, **kwargs) + self.status.choices = [(item.name, item.value) for item in PeriodStatus] + + diff --git a/eveai_app/views/entitlements_views.py b/eveai_app/views/entitlements_views.py index dccc7f3..bbf9cc3 100644 --- a/eveai_app/views/entitlements_views.py +++ b/eveai_app/views/entitlements_views.py @@ -13,11 +13,11 @@ from common.services.user import PartnerServices from common.services.user import UserServices from common.utils.eveai_exceptions import EveAIException from common.utils.security_utils import current_user_has_role -from .entitlements_forms import LicenseTierForm, LicenseForm +from .entitlements_forms import LicenseTierForm, LicenseForm, EditPeriodForm from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed from common.utils.nginx_utils import prefixed_url_for from common.utils.document_utils import set_logging_information, update_logging_information -from .list_views.entitlement_list_views import get_license_tiers_list_view, get_license_list_view +from .list_views.entitlement_list_views import get_license_tiers_list_view, get_license_list_view, get_license_periods_list_view from .list_views.list_view_utils import render_list_view entitlements_bp = Blueprint('entitlements_bp', __name__, url_prefix='/entitlements') @@ -255,14 +255,14 @@ def handle_license_selection(): case 'edit_license': return redirect(prefixed_url_for('entitlements_bp.edit_license', license_id=license_id)) case 'view_periods': - return redirect(prefixed_url_for('entitlements_bp.view_license_periods', license_id=license_id)) + return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id)) case _: return redirect(prefixed_url_for('entitlements_bp.licenses')) @entitlements_bp.route('/license//periods') @roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') -def view_license_periods(license_id): +def license_periods(license_id): license = License.query.get_or_404(license_id) # Verify user can access this license @@ -272,48 +272,77 @@ def view_license_periods(license_id): flash('Access denied to this license', 'danger') return redirect(prefixed_url_for('entitlements_bp.licenses')) - # Get all periods for this license - periods = (LicensePeriod.query - .filter_by(license_id=license_id) - .order_by(LicensePeriod.period_number) - .all()) + config = get_license_periods_list_view(license_id) - # Group related data for easy template access - usage_by_period = {} - payments_by_period = {} - invoices_by_period = {} + # Check if there was an error in getting the configuration + if config.get('error'): + return render_template("index.html") - for period in periods: - usage_by_period[period.id] = period.license_usage - payments_by_period[period.id] = list(period.payments) - invoices_by_period[period.id] = list(period.invoices) - - return render_template('entitlements/license_periods.html', - license=license, - periods=periods, - usage_by_period=usage_by_period, - payments_by_period=payments_by_period, - invoices_by_period=invoices_by_period) + return render_list_view('list_view.html', **config) -@entitlements_bp.route('/license//periods//transition', methods=['POST']) -@roles_accepted('Super User', 'Partner Admin') -def transition_period_status(license_id, period_id): +@entitlements_bp.route('/license_period/', methods=['GET', 'POST']) +@roles_accepted('Super User') +def edit_license_period(period_id): """Handle status transitions for license periods""" period = LicensePeriod.query.get_or_404(period_id) - new_status = request.form.get('new_status') + form = EditPeriodForm(obj=period) - try: - period.transition_status(PeriodStatus[new_status], current_user.id) - db.session.commit() - flash(f'Period {period.period_number} status updated to {new_status}', 'success') - except ValueError as e: - flash(f'Invalid status transition: {str(e)}', 'danger') - except Exception as e: - db.session.rollback() - flash(f'Error updating status: {str(e)}', 'danger') + if request.method == 'POST' and form.validate_on_submit(): + form.populate_obj(period) + update_logging_information(period, dt.now(tz.utc)) + match form.status.data: + case 'UPCOMING': + period.upcoming_at = dt.now(tz.utc) + case 'PENDING': + period.pending_at = dt.now(tz.utc) + case 'ACTIVE': + period.active_at = dt.now(tz.utc) + case 'COMPLETED': + period.completed_at = dt.now(tz.utc) + case 'INVOICED': + period.invoiced_at = dt.now(tz.utc) + case 'CLOSED': + period.closed_at = dt.now(tz.utc) - return redirect(prefixed_url_for('entitlements_bp.view_license_periods', license_id=license_id)) + try: + db.session.add(period) + db.session.commit() + flash('Period updated successfully.', 'success') + current_app.logger.info(f"Successfully updated period {period_id}") + except SQLAlchemyError as e: + db.session.rollback() + flash(f'Error updating status: {str(e)}', 'danger') + current_app.logger.error(f"Error updating period {period_id}: {str(e)}") + + return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=period.license_id)) + + return render_template('entitlements/edit_license_period.html', form=form) + + +@entitlements_bp.route('/license//handle_period_selection', methods=['POST']) +@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') +def handle_license_period_selection(license_id): + """Handle actions for license periods""" + action = request.form['action'] + + # For actions that don't require a selection + if 'selected_row' not in request.form: + return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id)) + + period_identification = request.form['selected_row'] + period_id = ast.literal_eval(period_identification).get('value') + + match action: + case 'view_period_details': + # TODO: Implement period details view if needed + flash('Period details view not yet implemented', 'info') + return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id)) + case 'edit_license_period': + # Display a form to choose the new status + return redirect(prefixed_url_for('entitlements_bp.edit_license_period', period_id=period_id)) + case _: + return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id)) @entitlements_bp.route('/view_licenses') diff --git a/eveai_app/views/list_views/entitlement_list_views.py b/eveai_app/views/list_views/entitlement_list_views.py index d633fd9..31b64bb 100644 --- a/eveai_app/views/list_views/entitlement_list_views.py +++ b/eveai_app/views/list_views/entitlement_list_views.py @@ -3,7 +3,7 @@ from datetime import datetime as dt, timezone as tz from flask import flash from sqlalchemy import or_, desc -from common.models.entitlements import LicenseTier, License +from common.models.entitlements import LicenseTier, License, LicensePeriod from common.services.user import PartnerServices, UserServices from common.utils.eveai_exceptions import EveAIException from common.utils.security_utils import current_user_has_role @@ -77,10 +77,16 @@ def get_license_tiers_list_view(): 'requiresSelection': False} ] - # Add assign license action if user has permission + # Add assign license actions if user has permission + current_app.logger.debug(f"Adding specific buttons") if UserServices.can_user_assign_license(): - actions.insert(1, {'value': 'assign_license', 'text': 'Assign License', 'class': 'btn-info', - 'requiresSelection': True}) + current_app.logger.debug(f"Adding Create License for Tenant") + actions.insert(1, {'value': 'create_license_for_tenant', 'text': 'Create License for Tenant', + 'class': 'btn-secondary', 'requiresSelection': True}) + if current_user_has_role('Super User'): + current_app.logger.debug(f"Adding Associate License Tier to Partner") + actions.insert(2, {'value': 'associate_license_tier_to_partner', 'text': 'Associate License Tier to Partner', + 'class': 'btn-secondary','requiresSelection': True}) # Initial sort configuration initial_sort = [{'column': 'start_date', 'dir': 'desc'}, {'column': 'id', 'dir': 'asc'}] @@ -139,7 +145,7 @@ def get_license_list_view(): {'title': 'License Tier', 'field': 'license_tier_name'}, {'title': 'Start Date', 'field': 'start_date', 'width': 120}, {'title': 'Nr of Periods', 'field': 'nr_of_periods', 'width': 120}, - {'title': 'Active', 'field': 'active', 'formatter': 'tickCross', 'width': 100} + {'title': 'Active', 'field': 'active', 'formatter': 'tickCross', 'width': 80} ] # Action definitions @@ -162,3 +168,74 @@ def get_license_list_view(): 'description': 'View and manage licenses', 'table_height': 700 } + + +def get_license_periods_list_view(license_id): + """Generate the license periods list view configuration""" + # Get the license object + license = License.query.get_or_404(license_id) + + # Get all periods for this license + periods = (LicensePeriod.query + .filter_by(license_id=license_id) + .order_by(LicensePeriod.period_number) + .all()) + + # Prepare data for Tabulator + data = [] + for period in periods: + # Get usage data + usage = period.license_usage + storage_used = usage.storage_mb_used if usage else 0 + embedding_used = usage.embedding_mb_used if usage else 0 + interaction_used = usage.interaction_total_tokens_used if usage else 0 + + # Get payment status + prepaid_payment = period.prepaid_payment + prepaid_status = prepaid_payment.status.name if prepaid_payment else 'N/A' + + # Get invoice status + prepaid_invoice = period.prepaid_invoice + invoice_status = prepaid_invoice.status.name if prepaid_invoice else 'N/A' + + data.append({ + 'id': period.id, + 'period_number': period.period_number, + 'period_start': period.period_start.strftime('%Y-%m-%d'), + 'period_end': period.period_end.strftime('%Y-%m-%d'), + 'status': period.status.name, + }) + + # Column definitions + columns = [ + {'title': 'ID', 'field': 'id', 'width': 60, 'type': 'number'}, + {'title': 'Period', 'field': 'period_number'}, + {'title': 'Start Date', 'field': 'period_start'}, + {'title': 'End Date', 'field': 'period_end'}, + {'title': 'Status', 'field': 'status'}, + ] + + # Action definitions + actions = [ + {'value': 'view_period_details', 'text': 'View Details', 'class': 'btn-primary', 'requiresSelection': True} + ] + + # If user has admin roles, add transition action + if current_user_has_role('Super User'): + actions.append({'value': 'edit_license_period', 'text': 'Edit Period', 'class': 'btn-secondary', 'requiresSelection': True}) + + # Initial sort configuration + initial_sort = [{'column': 'period_number', 'dir': 'asc'}] + + return { + 'title': 'License Periods', + 'data': data, + 'columns': columns, + 'actions': actions, + 'initial_sort': initial_sort, + 'table_id': 'license_periods_table', + 'form_action': url_for('entitlements_bp.handle_license_period_selection', license_id=license_id), + 'description': f'View and manage periods for License {license_id}', + 'table_height': 700, + 'license': license + } diff --git a/eveai_app/views/list_views/partner_list_views.py b/eveai_app/views/list_views/partner_list_views.py index 739be7f..f714fb8 100644 --- a/eveai_app/views/list_views/partner_list_views.py +++ b/eveai_app/views/list_views/partner_list_views.py @@ -10,9 +10,7 @@ def get_partners_list_view(): # Haal alle partners op met hun tenant informatie query = (db.session.query( Partner.id, - Partner.code, Partner.active, - Partner.logo_url, Tenant.name.label('name') ).join(Tenant, Partner.tenant_id == Tenant.id).order_by(Partner.id)) @@ -24,7 +22,6 @@ def get_partners_list_view(): for partner in all_partners: data.append({ 'id': partner.id, - 'code': partner.code, 'name': partner.name, 'active': partner.active }) @@ -32,7 +29,6 @@ def get_partners_list_view(): # Kolomdefinities columns = [ {'title': 'ID', 'field': 'id', 'width': 80}, - {'title': 'Code', 'field': 'code'}, {'title': 'Name', 'field': 'name'}, {'title': 'Active', 'field': 'active', 'formatter': 'tickCross'} ] diff --git a/eveai_app/views/list_views/user_list_views.py b/eveai_app/views/list_views/user_list_views.py index 18e2dea..9cefd87 100644 --- a/eveai_app/views/list_views/user_list_views.py +++ b/eveai_app/views/list_views/user_list_views.py @@ -3,7 +3,7 @@ from flask_security import roles_accepted from sqlalchemy.exc import SQLAlchemyError import ast -from common.models.user import Tenant, User, TenantDomain, TenantProject, TenantMake +from common.models.user import Tenant, User, TenantDomain, TenantProject, TenantMake, PartnerTenant, PartnerService from common.services.user import UserServices from eveai_app.views.list_views.list_view_utils import render_list_view @@ -232,3 +232,47 @@ def get_tenant_makes_list_view(tenant_id): 'form_action': url_for('user_bp.handle_tenant_make_selection'), 'description': f'Makes for tenant {tenant_id}' } + + + # Tenant Partner Services list view helper +def get_tenant_partner_services_list_view(tenant_id): + """Generate the tenant partner services list view configuration for a specific tenant""" + # Get partner services for the tenant through PartnerTenant association + query = PartnerService.query.join(PartnerTenant).filter(PartnerTenant.tenant_id == tenant_id) + partner_services = query.all() + + # Prepare data for Tabulator + data = [] + for service in partner_services: + data.append({ + 'id': service.id, + 'name': service.name, + 'type': service.type, + 'type_version': service.type_version, + 'active': service.active + }) + + # Column Definitions + columns = [ + {'title': 'ID', 'field': 'id', 'width': 80}, + {'title': 'Name', 'field': 'name'}, + {'title': 'Type', 'field': 'type'}, + {'title': 'Version', 'field': 'type_version'}, + {'title': 'Active', 'field': 'active', 'formatter': 'tickCross', 'width': 120} + ] + + # No actions needed as specified in requirements + actions = [] + + initial_sort = [{'column': 'name', 'dir': 'asc'}] + + return { + 'title': 'Partner Services', + 'data': data, + 'columns': columns, + 'actions': actions, + 'initial_sort': initial_sort, + 'table_id': 'tenant_partner_services_table', + 'form_action': url_for('user_bp.tenant_partner_services'), + 'description': f'Partner Services for tenant {tenant_id}' + } diff --git a/eveai_app/views/user_views.py b/eveai_app/views/user_views.py index 3ca34c7..0f822db 100644 --- a/eveai_app/views/user_views.py +++ b/eveai_app/views/user_views.py @@ -24,7 +24,8 @@ from common.services.user import UserServices from common.utils.mail_utils import send_email from eveai_app.views.list_views.user_list_views import get_tenants_list_view, get_users_list_view, \ - get_tenant_domains_list_view, get_tenant_projects_list_view, get_tenant_makes_list_view + get_tenant_domains_list_view, get_tenant_projects_list_view, get_tenant_makes_list_view, \ + get_tenant_partner_services_list_view from eveai_app.views.list_views.list_view_utils import render_list_view user_bp = Blueprint('user_bp', __name__, url_prefix='/user') @@ -686,6 +687,15 @@ def handle_tenant_make_selection(): return redirect(prefixed_url_for('user_bp.tenant_makes')) +@user_bp.route('/tenant_partner_services', methods=['GET', 'POST']) +@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin') +def tenant_partner_services(): + tenant_id = session['tenant']['id'] + config = get_tenant_partner_services_list_view(tenant_id) + return render_list_view('list_view.html', **config) + + + def reset_uniquifier(user): security.datastore.set_uniquifier(user) db.session.add(user) diff --git a/eveai_chat_workers/retrievers/globals/PARTNER_RAG/1_0.py b/eveai_chat_workers/retrievers/globals/PARTNER_RAG/1_0.py new file mode 100644 index 0000000..e5552d8 --- /dev/null +++ b/eveai_chat_workers/retrievers/globals/PARTNER_RAG/1_0.py @@ -0,0 +1,148 @@ +# retrievers/standard_rag.py +import json +from datetime import datetime as dt, timezone as tz +from typing import Dict, Any, List +from sqlalchemy import func, or_, desc, text +from sqlalchemy.exc import SQLAlchemyError +from flask import current_app + +from common.extensions import db +from flask_sqlalchemy import SQLAlchemy +from common.models.document import Document, DocumentVersion, Catalog, Retriever +from common.models.user import Tenant, PartnerRAGRetriever +from common.utils.datetime_utils import get_date_in_timezone +from common.utils.model_utils import get_embedding_model_and_class +from eveai_chat_workers.retrievers.base_retriever import BaseRetriever + +from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments, RetrieverResult, RetrieverMetadata + + +class RetrieverExecutor(BaseRetriever): + """Standard RAG retriever implementation""" + + def __init__(self, tenant_id: int, retriever_id: int, partner_id: int): + super().__init__(tenant_id, retriever_id) + self.partner_id = partner_id + self.partner_db = SQLAlchemy() + self._init_partner_db() + + def _init_partner_db(self): + self.partner_db.init_app(current_app) + self.partner_db.session.execute(text(f'set search_path to "{str(self.tenant_id)}", public')) + self.partner_db.session.commit() + + @property + def type(self) -> str: + return "PARTNER_RAG" + + @property + def type_version(self) -> str: + return "1.0" + + def retrieve(self, arguments: RetrieverArguments) -> List[RetrieverResult]: + """ + Retrieve documents based on query + + Args: + arguments: Validated RetrieverArguments containing at minimum: + - query: str - The search query + + Returns: + List[RetrieverResult]: List of retrieved documents with similarity scores + """ + try: + question = arguments.question + + # Get query embedding + query_embedding = self.embedding_model.embed_query(question) + + # Get the appropriate embedding database model + db_class = self.embedding_model_class + + # Get the current date for validity checks + current_date = dt.now(tz=tz.utc).date() + + # Create subquery for latest versions + subquery = ( + self.partner_db.session.query( + DocumentVersion.doc_id, + func.max(DocumentVersion.id).label('latest_version_id') + ) + .group_by(DocumentVersion.doc_id) + .subquery() + ) + + similarity_threshold = self.retriever.configuration.get('es_similarity_threshold', 0.3) + k = self.retriever.configuration.get('es_k', 8) + + # Main query + query_obj = ( + self.partner_db.session.query( + db_class, + DocumentVersion.url, + (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity') + ) + .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id) + .join(Document, DocumentVersion.doc_id == Document.id) + .join(subquery, DocumentVersion.id == subquery.c.latest_version_id) + .filter( + or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date), + or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date), + (1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold, + Document.catalog_id == self.catalog_id + ) + .order_by(desc('similarity')) + .limit(k) + ) + + results = query_obj.all() + + # Transform results into standard format + processed_results = [] + for doc, url, similarity in results: + # Parse user_metadata to ensure it's a dictionary + user_metadata = self._parse_metadata(doc.document_version.user_metadata) + processed_results.append( + RetrieverResult( + id=doc.id, + chunk=doc.chunk, + similarity=float(similarity), + metadata=RetrieverMetadata( + document_id=doc.document_version.doc_id, + version_id=doc.document_version.id, + document_name=f"Partner {self.partner_id} with Tenant ID {self.tenant_id} Doc: " + f"{doc.document_version.document.name}", + url=url or "", + user_metadata=user_metadata, + ) + ) + ) + + # Log the retrieval + if self.tuning: + compiled_query = str(query_obj.statement.compile( + compile_kwargs={"literal_binds": True} # This will include the actual values in the SQL + )) + self.log_tuning('retrieve', { + "arguments": arguments.model_dump(), + "similarity_threshold": similarity_threshold, + "k": k, + "query": compiled_query, + "Raw Results": str(results), + "Processed Results": [r.model_dump() for r in processed_results], + }) + + return processed_results + + except SQLAlchemyError as e: + current_app.logger.error(f'Error in RAG retrieval: {e}') + self.partner_db.session.rollback() + raise + except Exception as e: + current_app.logger.error(f'Unexpected error in RAG retrieval: {e}') + raise + finally: + if self.partner_db.session: + self.partner_db.session.close() + + diff --git a/eveai_chat_workers/specialists/globals/STANDARD_RAG_SPECIALIST/__init__.py b/eveai_chat_workers/retrievers/globals/PARTNER_RAG/__init__.py similarity index 100% rename from eveai_chat_workers/specialists/globals/STANDARD_RAG_SPECIALIST/__init__.py rename to eveai_chat_workers/retrievers/globals/PARTNER_RAG/__init__.py diff --git a/eveai_chat_workers/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1_0.py b/eveai_chat_workers/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1_0.py new file mode 100644 index 0000000..c4127a0 --- /dev/null +++ b/eveai_chat_workers/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1_0.py @@ -0,0 +1,243 @@ +import json +from os import wait +from typing import Optional, List, Dict, Any + +from crewai.flow.flow import start, listen, and_ +from flask import current_app +from pydantic import BaseModel, Field + +from common.extensions import db +from common.models.interaction import SpecialistRetriever +from common.models.user import Partner, PartnerService, PartnerTenant, PartnerRAGRetriever +from common.services.utils.translation_services import TranslationServices +from common.utils.business_event_context import current_event +from eveai_chat_workers.retrievers.base_retriever import BaseRetriever, get_retriever_class +from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments +from eveai_chat_workers.specialists.crewai_base_specialist import CrewAIBaseSpecialistExecutor +from eveai_chat_workers.specialists.specialist_typing import SpecialistResult, SpecialistArguments +from eveai_chat_workers.outputs.globals.rag.rag_v1_0 import RAGOutput +from eveai_chat_workers.specialists.crewai_base_classes import EveAICrewAICrew, EveAICrewAIFlow, EveAIFlowState + +INSUFFICIENT_INFORMATION_MESSAGE = ( + "We do not have the necessary information to provide you with the requested answers. " + "Please accept our apologies. Don't hesitate to ask other questions, and I'll do my best to answer them.") + + +class SpecialistExecutor(CrewAIBaseSpecialistExecutor): + """ + type: RAG_SPECIALIST + type_version: 1.0 + RAG Specialist Executor class + """ + + def __init__(self, tenant_id, specialist_id, session_id, task_id, **kwargs): + self.rag_crew = None + + super().__init__(tenant_id, specialist_id, session_id, task_id) + + @property + def type(self) -> str: + return "RAG_SPECIALIST" + + @property + def type_version(self) -> str: + return "1.1" + + def _initialize_retrievers(self) -> List[BaseRetriever]: + """Initialize all retrievers associated with this specialist""" + retrievers = [] + + partner_ids = ( + db.session.query(Partner.id) + .join(PartnerService, Partner.id == PartnerService.partner_id) + .join(PartnerTenant, PartnerService.id == PartnerTenant.partner_service_id) + .filter(PartnerTenant.tenant_id == self.tenant_id) + .distinct() + .all() + ) + + # Extract the actual partner IDs from the query result + partner_ids_list = [partner_id[0] for partner_id in partner_ids] + + # Get all corresponding PartnerRagRetrievers for the partner_ids list + partner_rag_retrievers = ( + PartnerRAGRetriever.query + .filter(PartnerRAGRetriever.partner_id.in_(partner_ids_list)) + .filter(PartnerRAGRetriever.tenant_id == self.tenant_id) + .all() + ) + + retriever_executor_class = get_retriever_class("PARTNER_RAG", "1.0") + # Get retriever associations from database + + self.log_tuning("_initialize_retrievers", {"Nr of partner retrievers": len(partner_rag_retrievers)}) + + for partner_rag_retriever in partner_rag_retrievers : + # Get retriever configuration from database + self.log_tuning("_initialize_retrievers", { + "Partner id": partner_rag_retriever.partner_id, + "Tenant id": partner_rag_retriever.tenant_id, + "Retriever id": partner_rag_retriever.retriever_id, + }) + + retriever_executor = retriever_executor_class(partner_rag_retriever.tenant_id, + partner_rag_retriever.retriever_id) + # Initialize retriever with its configuration + retrievers.append(retriever_executor) + + return retrievers + + def _config_task_agents(self): + self._add_task_agent("rag_task", "rag_agent") + + def _config_pydantic_outputs(self): + self._add_pydantic_output("rag_task", RAGOutput, "rag_output") + + def _config_state_result_relations(self): + self._add_state_result_relation("rag_output") + self._add_state_result_relation("citations") + + def _instantiate_specialist(self): + verbose = self.tuning + + rag_agents = [self.rag_agent] + rag_tasks = [self.rag_task] + self.rag_crew = EveAICrewAICrew( + self, + "Rag Crew", + agents=rag_agents, + tasks=rag_tasks, + verbose=verbose, + ) + + self.flow = RAGFlow( + self, + self.rag_crew, + ) + + def execute(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult: + self.log_tuning("RAG Specialist execution started", {}) + + if not self._cached_session.interactions: + specialist_phase = "initial" + else: + specialist_phase = self._cached_session.interactions[-1].specialist_results.get('phase', 'initial') + + results = None + + match specialist_phase: + case "initial": + results = self.execute_initial_state(arguments, formatted_context, citations) + case "rag": + results = self.execute_rag_state(arguments, formatted_context, citations) + + self.log_tuning(f"RAG Specialist execution ended", {"Results": results.model_dump()}) + + return results + + def execute_initial_state(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult: + self.log_tuning("RAG Specialist initial_state execution started", {}) + + welcome_message = self.specialist.configuration.get('welcome_message', 'Welcome! You can start asking questions') + welcome_message = TranslationServices.translate(self.tenant_id, welcome_message, arguments.language) + + self.flow.state.answer = welcome_message + self.flow.state.phase = "rag" + + results = RAGSpecialistResult.create_for_type(self.type, self.type_version) + + return results + + def execute_rag_state(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult: + self.log_tuning("RAG Specialist rag_state execution started", {}) + + insufficient_info_message = TranslationServices.translate(self.tenant_id, + INSUFFICIENT_INFORMATION_MESSAGE, + arguments.language) + + formatted_context, citations = self._retrieve_context(arguments) + + if formatted_context: + flow_inputs = { + "language": arguments.language, + "question": arguments.question, + "context": formatted_context, + "history": self.formatted_history, + "name": self.specialist.configuration.get('name', ''), + "welcome_message": self.specialist.configuration.get('welcome_message', '') + } + + flow_results = self.flow.kickoff(inputs=flow_inputs) + + if flow_results.rag_output.insufficient_info: + flow_results.rag_output.answer = insufficient_info_message + + rag_output = flow_results.rag_output + else: + rag_output = RAGOutput(answer=insufficient_info_message, insufficient_info=True) + + self.flow.state.rag_output = rag_output + self.flow.state.citations = citations + self.flow.state.answer = rag_output.answer + self.flow.state.phase = "rag" + + results = RAGSpecialistResult.create_for_type(self.type, self.type_version) + + return results + + +class RAGSpecialistInput(BaseModel): + language: Optional[str] = Field(None, alias="language") + question: Optional[str] = Field(None, alias="question") + context: Optional[str] = Field(None, alias="context") + history: Optional[str] = Field(None, alias="history") + name: Optional[str] = Field(None, alias="name") + welcome_message: Optional[str] = Field(None, alias="welcome_message") + + +class RAGSpecialistResult(SpecialistResult): + rag_output: Optional[RAGOutput] = Field(None, alias="Rag Output") + + +class RAGFlowState(EveAIFlowState): + """Flow state for RAG specialist that automatically updates from task outputs""" + input: Optional[RAGSpecialistInput] = None + rag_output: Optional[RAGOutput] = None + citations: Optional[List[Dict[str, Any]]] = None + + +class RAGFlow(EveAICrewAIFlow[RAGFlowState]): + def __init__(self, + specialist_executor: CrewAIBaseSpecialistExecutor, + rag_crew: EveAICrewAICrew, + **kwargs): + super().__init__(specialist_executor, "RAG Specialist Flow", **kwargs) + self.specialist_executor = specialist_executor + self.rag_crew = rag_crew + self.exception_raised = False + + @start() + def process_inputs(self): + return "" + + @listen(process_inputs) + async def execute_rag(self): + inputs = self.state.input.model_dump() + try: + crew_output = await self.rag_crew.kickoff_async(inputs=inputs) + self.specialist_executor.log_tuning("RAG Crew Output", crew_output.model_dump()) + output_pydantic = crew_output.pydantic + if not output_pydantic: + raw_json = json.loads(crew_output.raw) + output_pydantic = RAGOutput.model_validate(raw_json) + self.state.rag_output = output_pydantic + return crew_output + except Exception as e: + current_app.logger.error(f"CREW rag_crew Kickoff Error: {str(e)}") + self.exception_raised = True + raise e + + async def kickoff_async(self, inputs=None): + self.state.input = RAGSpecialistInput.model_validate(inputs) + result = await super().kickoff_async(inputs) + return self.state diff --git a/eveai_chat_workers/specialists/evie_partner/PARTNER_RAG_SPECIALIST/__init__.py b/eveai_chat_workers/specialists/evie_partner/PARTNER_RAG_SPECIALIST/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eveai_chat_workers/specialists/globals/STANDARD_RAG_SPECIALIST/1_0.py b/eveai_chat_workers/specialists/globals/STANDARD_RAG_SPECIALIST/1_0.py deleted file mode 100644 index 13a586b..0000000 --- a/eveai_chat_workers/specialists/globals/STANDARD_RAG_SPECIALIST/1_0.py +++ /dev/null @@ -1,239 +0,0 @@ -from datetime import datetime -from typing import Dict, Any, List -from flask import current_app -from langchain_core.exceptions import LangChainException -from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnableParallel, RunnablePassthrough - -from common.langchain.outputs.base import OutputRegistry -from common.langchain.outputs.rag import RAGOutput -from common.utils.business_event_context import current_event -from eveai_chat_workers.specialists.specialist_typing import SpecialistArguments, SpecialistResult -from eveai_chat_workers.chat_session_cache import get_chat_history -from common.models.interaction import Specialist -from common.utils.model_utils import create_language_template, replace_variable_in_template, \ - get_template -from eveai_chat_workers.specialists.base_specialist import BaseSpecialistExecutor -from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments - - -class SpecialistExecutor(BaseSpecialistExecutor): - """ - type: STANDARD_RAG - type_version: 1.0 - Standard Q&A RAG Specialist implementation that combines retriever results - with LLM processing to generate answers. - """ - - def __init__(self, tenant_id: int, specialist_id: int, session_id: str, task_id: str): - super().__init__(tenant_id, specialist_id, session_id, task_id) - - # Check and load the specialist - specialist = Specialist.query.get_or_404(specialist_id) - # Set the specific configuration for the RAG Specialist - self.specialist_context = specialist.configuration.get('specialist_context', '') - self.temperature = specialist.configuration.get('temperature', 0.3) - self.tuning = specialist.tuning - - # Initialize retrievers - self.retrievers = self._initialize_retrievers() - - @property - def type(self) -> str: - return "STANDARD_RAG_SPECIALIST" - - @property - def type_version(self) -> str: - return "1.0" - - @property - def required_templates(self) -> List[str]: - """List of required templates for this specialist""" - return ['rag', 'history'] - - def _detail_question(self, language: str, question: str) -> str: - """Detail question based on conversation history""" - try: - # Get cached session history - cached_session = get_chat_history(self.session_id) - - # Format history for the prompt - formatted_history = "\n\n".join([ - f"HUMAN:\n{interaction.specialist_results.get('detailed_query')}\n\n" - f"AI:\n{interaction.specialist_results.get('answer')}" - for interaction in cached_session.interactions - ]) - - # Get LLM and template - template, llm = get_template("history", temperature=0.3) - language_template = create_language_template(template, language) - - # Create prompt - history_prompt = ChatPromptTemplate.from_template(language_template) - - # Create chain - chain = ( - history_prompt | - llm | - StrOutputParser() - ) - - # Execute chain - detailed_question = chain.invoke({ - "history": formatted_history, - "question": question - }) - - if self.tuning: - self.log_tuning("_detail_question", { - "cached_session_id": cached_session.session_id, - "cached_session.interactions": str(cached_session.interactions), - "original_question": question, - "history_used": formatted_history, - "detailed_question": detailed_question, - }) - - return detailed_question - - except Exception as e: - current_app.logger.error(f"Error detailing question: {e}") - return question # Fallback to original question - - def execute(self, arguments: SpecialistArguments) -> SpecialistResult: - """ - Execute the RAG specialist to generate an answer - """ - start_time = datetime.now() - - try: - with current_event.create_span("Specialist Detail Question"): - self.update_progress("Detail Question Start", {}) - # Get required arguments - language = arguments.language - query = arguments.query - detailed_question = self._detail_question(language, query) - self.update_progress("Detail Question End", {}) - - # Log the start of retrieval process if tuning is enabled - with current_event.create_span("Specialist Retrieval"): - self.log_tuning("Starting context retrieval", { - "num_retrievers": len(self.retrievers), - "all arguments": arguments.model_dump(), - }) - self.update_progress("EveAI Retriever Start", {}) - - # Get retriever-specific arguments - retriever_arguments = arguments.retriever_arguments - - # Collect context from all retrievers - all_context = [] - for retriever in self.retrievers: - # Get arguments for this specific retriever - retriever_id = str(retriever.retriever_id) - if retriever_id not in retriever_arguments: - current_app.logger.error(f"Missing arguments for retriever {retriever_id}") - continue - - # Get the retriever's arguments and update the query - current_retriever_args = retriever_arguments[retriever_id] - if isinstance(retriever_arguments[retriever_id], RetrieverArguments): - updated_args = current_retriever_args.model_dump() - updated_args['query'] = detailed_question - retriever_args = RetrieverArguments(**updated_args) - else: - # Create a new RetrieverArguments instance from the dictionary - current_retriever_args['query'] = detailed_question - retriever_args = RetrieverArguments(**current_retriever_args) - - # Each retriever gets its own specific arguments - retriever_result = retriever.retrieve(retriever_args) - all_context.extend(retriever_result) - - # Sort by similarity if available and get unique contexts - all_context.sort(key=lambda x: x.similarity, reverse=True) - unique_contexts = [] - seen_chunks = set() - for ctx in all_context: - if ctx.chunk not in seen_chunks: - unique_contexts.append(ctx) - seen_chunks.add(ctx.chunk) - - self.log_tuning("Context retrieval completed", { - "total_contexts": len(all_context), - "unique_contexts": len(unique_contexts), - "average_similarity": sum(ctx.similarity for ctx in unique_contexts) / len( - unique_contexts) if unique_contexts else 0 - }) - self.update_progress("EveAI Retriever Complete", {}) - - # Prepare context for LLM - formatted_context = "\n\n".join([ - f"SOURCE: {ctx.metadata.document_id}\n\n{ctx.chunk}" - for ctx in unique_contexts - ]) - - with current_event.create_span("Specialist RAG invocation"): - try: - self.update_progress(self.task_id, "EveAI Chain Start", {}) - template, llm = get_template("rag", self.temperature) - language_template = create_language_template(template, language) - full_template = replace_variable_in_template( - language_template, - "{tenant_context}", - self.specialist_context - ) - - if self.tuning: - self.log_tuning("Template preparation completed", { - "template": full_template, - "context": formatted_context, - "tenant_context": self.specialist_context, - }) - - # Create prompt - rag_prompt = ChatPromptTemplate.from_template(full_template) - - # Setup chain components - setup_and_retrieval = RunnableParallel({ - "context": lambda x: formatted_context, - "question": lambda x: x - }) - - # Get output schema for structured output - output_schema = OutputRegistry.get_schema(self.type) - structured_llm = llm.with_structured_output(output_schema) - chain = setup_and_retrieval | rag_prompt | structured_llm - - raw_result = chain.invoke(detailed_question) - result = SpecialistResult.create_for_type( - self.type, - self.type_version, - detailed_question=detailed_question, - answer=raw_result.answer, - citations=[ctx.metadata.document_id for ctx in unique_contexts - if ctx.id in raw_result.citations], - insufficient_info=raw_result.insufficient_info - ) - - if self.tuning: - self.log_tuning("LLM chain execution completed", { - "Result": result.model_dump() - }) - self.update_progress("EveAI Chain Complete", {}) - - except Exception as e: - current_app.logger.error(f"Error in LLM processing: {e}") - if self.tuning: - self.log_tuning("LLM processing error", {"error": str(e)}) - raise - - return result - - except Exception as e: - current_app.logger.error(f'Error in RAG specialist execution: {str(e)}') - raise - - -# Register the specialist type -OutputRegistry.register("STANDARD_RAG_SPECIALIST", RAGOutput)