- Introduction of PARTNER_RAG retriever, PARTNER_RAG_SPECIALIST and linked Agent and Task, to support documentation inquiries in the management app (eveai_app)

- Addition of a tenant_partner_services view to show partner services from the viewpoint of a tenant
- Addition of domain model diagrams
- Addition of license_periods views and form

common/models/user.py

@@ -341,3 +341,16 @@ class TranslationCache(db.Model):
     updated_by = db.Column(db.Integer, db.ForeignKey('public.user.id'), nullable=True)
 
     last_used_at = db.Column(db.DateTime, nullable=True)
+
+
+class PartnerRAGRetriever(db.Model):
+    __bind_key__ = 'public'
+    __table_args__ = (
+        {'schema': 'public'},
+        db.PrimaryKeyConstraint('tenant_id', 'retriever_id'),
+        db.UniqueConstraint('partner_id', 'tenant_id', 'retriever_id')
+    )
+
+    partner_id = db.Column(db.Integer, db.ForeignKey('public.partner.id'), nullable=False)
+    tenant_id = db.Column(db.Integer, db.ForeignKey('public.tenant.id'), nullable=False)
+    retriever_id = db.Column(db.Integer, nullable=False)

common/services/entitlements/license_period_services.py

@@ -41,7 +41,7 @@ class LicensePeriodServices:
                 current_app.logger.debug(f"Found license period {license_period.id} for tenant {tenant_id} "
                                          f"with status {license_period.status}")
                 match license_period.status:
-                    case PeriodStatus.UPCOMING:
+                    case PeriodStatus.UPCOMING | PeriodStatus.PENDING:
                         current_app.logger.debug(f"In upcoming state")
                         LicensePeriodServices._complete_last_license_period(tenant_id=tenant_id)
                         current_app.logger.debug(f"Completed last license period for tenant {tenant_id}")
@@ -73,8 +73,6 @@ class LicensePeriodServices:
                                 raise EveAIPendingLicensePeriod()
                     case PeriodStatus.ACTIVE:
                         return license_period
-                    case PeriodStatus.PENDING:
-                        return license_period
             else:
                 raise EveAILicensePeriodsExceeded(license_id=None)
         except SQLAlchemyError as e:
@@ -125,7 +123,7 @@ class LicensePeriodServices:
             tenant_id=tenant_id,
             period_number=next_period_number,
             period_start=the_license.start_date + relativedelta(months=next_period_number-1),
-            period_end=the_license.end_date + relativedelta(months=next_period_number, days=-1),
+            period_end=the_license.start_date + relativedelta(months=next_period_number, days=-1),
             status=PeriodStatus.UPCOMING,
             upcoming_at=dt.now(tz.utc),
         )

config/agents/evie_partner/PARTNER_RAG_AGENT/1.0.0.yaml

@@ -0,0 +1,26 @@
+version: "1.0.0"
+name: "Partner Rag Agent"
+role: >
+  You are a virtual assistant responsible for answering user questions about the Evie platform (Ask Eve AI) and products 
+  developed by partners on top of it. You are reliable point of contact for end-users seeking help, clarification, or 
+  deeper understanding of features, capabilities, integrations, or workflows related to these AI-powered solutions.
+goal: >
+  Your primary goal is to:
+  	•	Provide clear, relevant, and accurate responses to user questions.
+  	•	Reduce friction in user onboarding and daily usage.
+  	•	Increase user confidence and adoption of both the platform and partner-developed products.
+  	•	Act as a bridge between documentation and practical application, enabling users to help themselves through intelligent guidance.
+backstory: >
+  You have availability Evie’s own documentation, partner product manuals, and real user interactions. You are designed 
+  to replace passive documentation with active, contextual assistance. 
+  You have evolved beyond a support bot: you combine knowledge, reasoning, and a friendly tone to act as a product 
+  companion that grows with the ecosystem. As partner products expand, the agent updates its knowledge and learns to 
+  distinguish between general platform capabilities and product-specific nuances, offering a personalised experience 
+  each time.
+full_model_name: "mistral.mistral-medium-latest"
+temperature: 0.3
+metadata:
+  author: "Josako"
+  date_added: "2025-07-16"
+  description: "An Agent that does RAG based on a user's question, RAG content & history"
+  changes: "Initial version"

config/partner_services/globals/KNOWLEDGE_SERVICE/1.0.0.yaml

@@ -0,0 +1,9 @@
+version: "1.0.0"
+name: "Knowledge Service"
+configuration: {}
+permissions: {}
+metadata:
+  author: "Josako"
+  date_added: "2025-04-02"
+  changes: "Initial version"
+  description: "Partner providing catalog content"

config/retrievers/evie_partner/PARTNER_RAG/1.0.0.yaml

@@ -0,0 +1,21 @@
+version: "1.0.0"
+name: "Standard RAG Retriever"
+configuration:
+  es_k:
+    name: "es_k"
+    type: "integer"
+    description: "K-value to retrieve embeddings (max embeddings retrieved)"
+    required: true
+    default: 8
+  es_similarity_threshold:
+    name: "es_similarity_threshold"
+    type: "float"
+    description: "Similarity threshold for retrieving embeddings"
+    required: true
+    default: 0.3
+arguments: {}
+metadata:
+  author: "Josako"
+  date_added: "2025-01-24"
+  changes: "Initial version"
+  description: "Retrieving all embeddings conform the query"

config/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1.0.0.yaml

@@ -0,0 +1,34 @@
+version: "1.0.0"
+name: "Partner RAG Specialist"
+framework: "crewai"
+chat: true
+configuration: {}
+arguments: {}
+results:
+  rag_output:
+    answer:
+      name: "answer"
+      type: "str"
+      description: "Answer to the query"
+      required: true
+    citations:
+      name: "citations"
+      type: "List[str]"
+      description: "List of citations"
+      required: false
+    insufficient_info:
+      name: "insufficient_info"
+      type: "bool"
+      description: "Whether or not the query is insufficient info"
+      required: true
+agents:
+  - type: "PARTNER_RAG_AGENT"
+    version: "1.0"
+tasks:
+  - type: "PARTNER_RAG_TASK"
+    version: "1.0"
+metadata:
+  author: "Josako"
+  date_added: "2025-07-16"
+  changes: "Initial version"
+  description: "Q&A through Partner RAG Specialist (for documentation purposes)"

config/specialists/globals/RAG_SPECIALIST/1.1.0.yaml

@@ -1,4 +1,4 @@
-version: "1.0.0"
+version: "1.1.0"
 name: "RAG Specialist"
 framework: "crewai"
 chat: true

config/specialists/globals/STANDARD_RAG_SPECIALIST/1.0.0.yaml

@@ -1,53 +0,0 @@
-version: 1.0.0
-name: "Standard RAG Specialist"
-framework: "langchain"
-chat: true
-configuration:
-  specialist_context:
-    name: "Specialist Context"
-    type: "text"
-    description: "The context to be used by the specialist."
-    required: false
-  temperature:
-    name: "Temperature"
-    type: "number"
-    description: "The inference temperature to be used by the specialist."
-    required: false
-    default: 0.3
-arguments:
-  language:
-    name: "Language"
-    type: "str"
-    description: "Language code to be used for receiving questions and giving answers"
-    required: true
-  query:
-    name: "query"
-    type: "str"
-    description: "Query to answer"
-    required: true
-results:
-  detailed_query:
-    name: "detailed_query"
-    type: "str"
-    description: "The query detailed with the Chat Session History."
-    required: true
-  answer:
-    name: "answer"
-    type: "str"
-    description: "Answer to the query"
-    required: true
-  citations:
-    name: "citations"
-    type: "List[str]"
-    description: "List of citations"
-    required: false
-  insufficient_info:
-    name: "insufficient_info"
-    type: "bool"
-    description: "Whether or not the query is insufficient info"
-    required: true
-metadata:
-  author: "Josako"
-  date_added: "2025-01-08"
-  changes: "Initial version"
-  description: "A Specialist that performs standard Q&A"

config/tasks/evie_partner/PARTNER_RAG_TASK/1.0.0.yaml

@@ -0,0 +1,22 @@
+version: "1.0.0"
+name: "RAG Task"
+task_description: >
+  Answer the question based on the following context, and taking into account the history of the discussion. Try not to
+  repeat answers already given in the recent history,  unless confirmation is required or repetition is essential to 
+  give a coherent answer.
+  Answer the end user in the language used in his/her question.
+  If the question cannot be answered using the given context, answer "I have insufficient information to answer this 
+  question."
+  Context (in between triple $):
+    $$${context}$$$
+  History (in between triple €):
+    €€€{history}€€€
+  Question (in between triple £):
+    £££{question}£££
+expected_output: >
+  Your answer.
+metadata:
+  author: "Josako"
+  date_added: "2025-07-16"
+  description: "A Task that gives RAG-based answers"
+  changes: "Initial version"

config/type_defs/partner_service_types.py

@@ -1,9 +1,5 @@
 # config/type_defs/partner_service_types.py
 PARTNER_SERVICE_TYPES = {
-    "REFERRAL_SERVICE": {
-        "name": "Referral Service",
-        "description": "Partner referring new customers",
-    },
     "KNOWLEDGE_SERVICE": {
         "name": "Knowledge Service",
         "description": "Partner providing catalog content",

config/type_defs/retriever_types.py

@@ -4,6 +4,11 @@ RETRIEVER_TYPES = {
         "name": "Standard RAG Retriever",
         "description": "Retrieving all embeddings from the catalog conform the query",
     },
+    "PARTNER_RAG": {
+        "name": "Partner RAG Retriever",
+        "description": "RAG intended for partner documentation",
+        "partner": "evie_partner"
+    },
     "TRAICIE_ROLE_DEFINITION_BY_ROLE_IDENTIFICATION": {
         "name": "Traicie Role Definition Retriever by Role Identification",
         "description": "Retrieves relevant role information for a given role",

config/type_defs/specialist_types.py

@@ -1,13 +1,14 @@
 # Specialist Types
 SPECIALIST_TYPES = {
-    "STANDARD_RAG_SPECIALIST": {
-        "name": "Standard RAG Specialist",
-        "description": "Standard Q&A through RAG Specialist",
-    },
     "RAG_SPECIALIST": {
         "name": "RAG Specialist",
         "description": "Q&A through RAG Specialist",
     },
+    "PARTNER_RAG_SPECIALIST": {
+        "name": "Partner RAG Specialist",
+        "description": "Q&A through Partner RAG Specialist (for documentation purposes)",
+        "partner": "evie_partner"
+    },
     "SPIN_SPECIALIST": {
         "name": "Spin Sales Specialist",
         "description": "A specialist that allows to answer user queries, try to get SPIN-information and Identification",

documentation/document_domain.mermaid

@@ -0,0 +1,138 @@
+erDiagram
+    CATALOG {
+        int id PK
+        string name
+        text description
+        string type
+        string type_version
+        int min_chunk_size
+        int max_chunk_size
+        jsonb user_metadata
+        jsonb system_metadata
+        jsonb configuration
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PROCESSOR {
+        int id PK
+        string name
+        text description
+        int catalog_id FK
+        string type
+        string sub_file_type
+        boolean active
+        boolean tuning
+        jsonb user_metadata
+        jsonb system_metadata
+        jsonb configuration
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    RETRIEVER {
+        int id PK
+        string name
+        text description
+        int catalog_id FK
+        string type
+        string type_version
+        boolean tuning
+        jsonb user_metadata
+        jsonb system_metadata
+        jsonb configuration
+        jsonb arguments
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    DOCUMENT {
+        int id PK
+        int catalog_id FK
+        string name
+        datetime valid_from
+        datetime valid_to
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    DOCUMENT_VERSION {
+        int id PK
+        int doc_id FK
+        string url
+        string bucket_name
+        string object_name
+        string file_type
+        string sub_file_type
+        float file_size
+        string language
+        text user_context
+        text system_context
+        jsonb user_metadata
+        jsonb system_metadata
+        jsonb catalog_properties
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+        boolean processing
+        datetime processing_started_at
+        datetime processing_finished_at
+        string processing_error
+    }
+
+    EMBEDDING {
+        int id PK
+        string type
+        int doc_vers_id FK
+        boolean active
+        text chunk
+    }
+
+    EMBEDDING_MISTRAL {
+        int id PK,FK
+        vector_1024 embedding
+    }
+
+    EMBEDDING_SMALL_OPENAI {
+        int id PK,FK
+        vector_1536 embedding
+    }
+
+    EMBEDDING_LARGE_OPENAI {
+        int id PK,FK
+        vector_3072 embedding
+    }
+
+    USER {
+        int id PK
+        string user_name
+        string email
+    }
+
+    %% Relationships
+    CATALOG ||--o{ PROCESSOR : "has many"
+    CATALOG ||--o{ RETRIEVER : "has many"
+    CATALOG ||--o{ DOCUMENT : "has many"
+    
+    DOCUMENT ||--o{ DOCUMENT_VERSION : "has many"
+    
+    DOCUMENT_VERSION ||--o{ EMBEDDING : "has many"
+    
+    EMBEDDING ||--o| EMBEDDING_MISTRAL : "inheritance"
+    EMBEDDING ||--o| EMBEDDING_SMALL_OPENAI : "inheritance"
+    EMBEDDING ||--o| EMBEDDING_LARGE_OPENAI : "inheritance"
+    
+    USER ||--o{ CATALOG : "creates/updates"
+    USER ||--o{ PROCESSOR : "creates/updates"
+    USER ||--o{ RETRIEVER : "creates/updates"
+    USER ||--o{ DOCUMENT : "creates/updates"
+    USER ||--o{ DOCUMENT_VERSION : "creates/updates"

documentation/entitlements_domain.mermaid

@@ -0,0 +1,244 @@
+erDiagram
+    BUSINESS_EVENT_LOG {
+        int id PK
+        datetime timestamp
+        string event_type
+        int tenant_id
+        string trace_id
+        string span_id
+        string span_name
+        string parent_span_id
+        int document_version_id
+        float document_version_file_size
+        int specialist_id
+        string specialist_type
+        string specialist_type_version
+        string chat_session_id
+        int interaction_id
+        string environment
+        int llm_metrics_total_tokens
+        int llm_metrics_prompt_tokens
+        int llm_metrics_completion_tokens
+        float llm_metrics_total_time
+        int llm_metrics_nr_of_pages
+        int llm_metrics_call_count
+        string llm_interaction_type
+        text message
+        int license_usage_id FK
+    }
+
+    LICENSE {
+        int id PK
+        int tenant_id FK
+        int tier_id FK
+        date start_date
+        date end_date
+        int nr_of_periods
+        string currency
+        boolean yearly_payment
+        float basic_fee
+        int max_storage_mb
+        float additional_storage_price
+        int additional_storage_bucket
+        int included_embedding_mb
+        decimal additional_embedding_price
+        int additional_embedding_bucket
+        int included_interaction_tokens
+        decimal additional_interaction_token_price
+        int additional_interaction_bucket
+        float overage_embedding
+        float overage_interaction
+        boolean additional_storage_allowed
+        boolean additional_embedding_allowed
+        boolean additional_interaction_allowed
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    LICENSE_TIER {
+        int id PK
+        string name
+        string version
+        date start_date
+        date end_date
+        float basic_fee_d
+        float basic_fee_e
+        int max_storage_mb
+        decimal additional_storage_price_d
+        decimal additional_storage_price_e
+        int additional_storage_bucket
+        int included_embedding_mb
+        decimal additional_embedding_price_d
+        decimal additional_embedding_price_e
+        int additional_embedding_bucket
+        int included_interaction_tokens
+        decimal additional_interaction_token_price_d
+        decimal additional_interaction_token_price_e
+        int additional_interaction_bucket
+        float standard_overage_embedding
+        float standard_overage_interaction
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PARTNER_SERVICE_LICENSE_TIER {
+        int partner_service_id PK,FK
+        int license_tier_id PK,FK
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    LICENSE_PERIOD {
+        int id PK
+        int license_id FK
+        int tenant_id FK
+        int period_number
+        date period_start
+        date period_end
+        string currency
+        float basic_fee
+        int max_storage_mb
+        float additional_storage_price
+        int additional_storage_bucket
+        int included_embedding_mb
+        decimal additional_embedding_price
+        int additional_embedding_bucket
+        int included_interaction_tokens
+        decimal additional_interaction_token_price
+        int additional_interaction_bucket
+        boolean additional_storage_allowed
+        boolean additional_embedding_allowed
+        boolean additional_interaction_allowed
+        enum status
+        datetime upcoming_at
+        datetime pending_at
+        datetime active_at
+        datetime completed_at
+        datetime invoiced_at
+        datetime closed_at
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    LICENSE_USAGE {
+        int id PK
+        int tenant_id FK
+        float storage_mb_used
+        float embedding_mb_used
+        int embedding_prompt_tokens_used
+        int embedding_completion_tokens_used
+        int embedding_total_tokens_used
+        int interaction_prompt_tokens_used
+        int interaction_completion_tokens_used
+        int interaction_total_tokens_used
+        int license_period_id FK
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PAYMENT {
+        int id PK
+        int license_period_id FK
+        int tenant_id FK
+        enum payment_type
+        decimal amount
+        string currency
+        text description
+        enum status
+        string external_payment_id
+        string payment_method
+        jsonb provider_data
+        datetime paid_at
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    INVOICE {
+        int id PK
+        int license_period_id FK
+        int payment_id FK
+        int tenant_id FK
+        enum invoice_type
+        string invoice_number
+        date invoice_date
+        date due_date
+        decimal amount
+        string currency
+        decimal tax_amount
+        text description
+        enum status
+        datetime sent_at
+        datetime paid_at
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    LICENSE_CHANGE_LOG {
+        int id PK
+        int license_id FK
+        datetime changed_at
+        string field_name
+        string old_value
+        string new_value
+        text reason
+        int created_by FK
+    }
+
+    TENANT {
+        int id PK
+        string name
+        string currency
+    }
+
+    USER {
+        int id PK
+        string user_name
+        string email
+    }
+
+    PARTNER_SERVICE {
+        int id PK
+        string name
+        string type
+    }
+
+    %% Main business relationships
+    TENANT ||--o{ LICENSE : "has many"
+    LICENSE_TIER ||--o{ LICENSE : "has many"
+    LICENSE ||--o{ LICENSE_PERIOD : "has many"
+    LICENSE_PERIOD ||--|| LICENSE_USAGE : "has one"
+    LICENSE_PERIOD ||--o{ PAYMENT : "has many"
+    LICENSE_PERIOD ||--o{ INVOICE : "has many"
+    
+    %% License management
+    LICENSE ||--o{ LICENSE_CHANGE_LOG : "has many"
+    
+    %% Payment-Invoice relationship
+    PAYMENT ||--o| INVOICE : "can have"
+    
+    %% Partner service licensing
+    PARTNER_SERVICE ||--o{ PARTNER_SERVICE_LICENSE_TIER : "has many"
+    LICENSE_TIER ||--o{ PARTNER_SERVICE_LICENSE_TIER : "has many"
+    
+    %% Event logging
+    LICENSE_USAGE ||--o{ BUSINESS_EVENT_LOG : "has many"
+    
+    %% Tenant relationships
+    TENANT ||--o{ LICENSE_PERIOD : "has many"
+    TENANT ||--o{ LICENSE_USAGE : "has many"
+    TENANT ||--o{ PAYMENT : "has many"
+    TENANT ||--o{ INVOICE : "has many"

documentation/interaction_domain.mermaid

@@ -0,0 +1,211 @@
+erDiagram
+    CHAT_SESSION {
+        int id PK
+        int user_id FK
+        string session_id
+        datetime session_start
+        datetime session_end
+        string timezone
+    }
+
+    SPECIALIST {
+        int id PK
+        string name
+        text description
+        string type
+        string type_version
+        boolean tuning
+        jsonb configuration
+        jsonb arguments
+        boolean active
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    EVE_AI_ASSET {
+        int id PK
+        string name
+        text description
+        string type
+        string type_version
+        string bucket_name
+        string object_name
+        string file_type
+        float file_size
+        jsonb user_metadata
+        jsonb system_metadata
+        jsonb configuration
+        int prompt_tokens
+        int completion_tokens
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+        datetime last_used_at
+    }
+
+    EVE_AI_AGENT {
+        int id PK
+        int specialist_id FK
+        string name
+        text description
+        string type
+        string type_version
+        text role
+        text goal
+        text backstory
+        boolean tuning
+        jsonb configuration
+        jsonb arguments
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    EVE_AI_TASK {
+        int id PK
+        int specialist_id FK
+        string name
+        text description
+        string type
+        string type_version
+        text task_description
+        text expected_output
+        boolean tuning
+        jsonb configuration
+        jsonb arguments
+        jsonb context
+        boolean asynchronous
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    EVE_AI_TOOL {
+        int id PK
+        int specialist_id FK
+        string name
+        text description
+        string type
+        string type_version
+        boolean tuning
+        jsonb configuration
+        jsonb arguments
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    DISPATCHER {
+        int id PK
+        string name
+        text description
+        string type
+        string type_version
+        boolean tuning
+        jsonb configuration
+        jsonb arguments
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    INTERACTION {
+        int id PK
+        int chat_session_id FK
+        int specialist_id FK
+        jsonb specialist_arguments
+        jsonb specialist_results
+        string timezone
+        int appreciation
+        datetime question_at
+        datetime detailed_question_at
+        datetime answer_at
+        string processing_error
+    }
+
+    INTERACTION_EMBEDDING {
+        int interaction_id PK,FK
+        int embedding_id PK,FK
+    }
+
+    SPECIALIST_RETRIEVER {
+        int specialist_id PK,FK
+        int retriever_id PK,FK
+    }
+
+    SPECIALIST_DISPATCHER {
+        int specialist_id PK,FK
+        int dispatcher_id PK,FK
+    }
+
+    SPECIALIST_MAGIC_LINK {
+        int id PK
+        string name
+        text description
+        int specialist_id FK
+        int tenant_make_id FK
+        string magic_link_code
+        datetime valid_from
+        datetime valid_to
+        jsonb specialist_args
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    USER {
+        int id PK
+        string user_name
+        string email
+    }
+
+    TENANT_MAKE {
+        int id PK
+        string name
+        text description
+    }
+
+    RETRIEVER {
+        int id PK
+        string name
+        text description
+    }
+
+    EMBEDDING {
+        int id PK
+        string type
+        text chunk
+    }
+
+    %% Main conversation flow
+    USER ||--o{ CHAT_SESSION : "has many"
+    CHAT_SESSION ||--o{ INTERACTION : "has many"
+    SPECIALIST ||--o{ INTERACTION : "processes"
+    
+    %% Specialist composition (EveAI components)
+    SPECIALIST ||--o{ EVE_AI_AGENT : "has many"
+    SPECIALIST ||--o{ EVE_AI_TASK : "has many"
+    SPECIALIST ||--o{ EVE_AI_TOOL : "has many"
+    
+    %% Specialist connections
+    SPECIALIST ||--o{ SPECIALIST_RETRIEVER : "uses retrievers"
+    RETRIEVER ||--o{ SPECIALIST_RETRIEVER : "used by specialists"
+    
+    SPECIALIST ||--o{ SPECIALIST_DISPATCHER : "uses dispatchers"
+    DISPATCHER ||--o{ SPECIALIST_DISPATCHER : "used by specialists"
+    
+    %% Interaction results
+    INTERACTION ||--o{ INTERACTION_EMBEDDING : "references embeddings"
+    EMBEDDING ||--o{ INTERACTION_EMBEDDING : "used in interactions"
+    
+    %% Magic links for specialist access
+    SPECIALIST ||--o{ SPECIALIST_MAGIC_LINK : "has magic links"
+    TENANT_MAKE ||--o{ SPECIALIST_MAGIC_LINK : "branded links"

documentation/user_domain.mermaid

@@ -0,0 +1,199 @@
+erDiagram
+    TENANT {
+        int id PK
+        string code
+        string name
+        string website
+        string timezone
+        string type
+        string currency
+        boolean storage_dirty
+        int default_tenant_make_id FK
+        datetime created_at
+        datetime updated_at
+    }
+
+    USER {
+        int id PK
+        int tenant_id FK
+        string user_name
+        string email
+        string password
+        string first_name
+        string last_name
+        boolean active
+        string fs_uniquifier
+        datetime confirmed_at
+        date valid_to
+        boolean is_primary_contact
+        boolean is_financial_contact
+        datetime last_login_at
+        datetime current_login_at
+        string last_login_ip
+        string current_login_ip
+        int login_count
+        datetime created_at
+        datetime updated_at
+    }
+
+    ROLE {
+        int id PK
+        string name
+        string description
+    }
+
+    ROLES_USERS {
+        int user_id PK,FK
+        int role_id PK,FK
+    }
+
+    TENANT_DOMAIN {
+        int id PK
+        int tenant_id FK
+        string domain
+        date valid_to
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    TENANT_PROJECT {
+        int id PK
+        int tenant_id FK
+        string name
+        text description
+        array services
+        string encrypted_api_key
+        string visual_api_key
+        boolean active
+        string responsible_email
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    TENANT_MAKE {
+        int id PK
+        int tenant_id FK
+        string name
+        text description
+        boolean active
+        string website
+        string logo_url
+        string default_language
+        array allowed_languages
+        jsonb chat_customisation_options
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PARTNER {
+        int id PK
+        int tenant_id FK
+        string code
+        string logo_url
+        boolean active
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PARTNER_SERVICE {
+        int id PK
+        int partner_id FK
+        string name
+        text description
+        string type
+        string type_version
+        boolean active
+        json configuration
+        json permissions
+        json system_metadata
+        json user_metadata
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PARTNER_TENANT {
+        int partner_service_id PK,FK
+        int tenant_id PK,FK
+        json configuration
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+    }
+
+    PARTNER_SERVICE_LICENSE_TIER {
+        int id PK
+        int partner_service_id FK
+    }
+
+    SPECIALIST_MAGIC_LINK_TENANT {
+        string magic_link_code PK
+        int tenant_id FK
+    }
+
+    TRANSLATION_CACHE {
+        string cache_key PK
+        text source_text
+        text translated_text
+        string source_language
+        string target_language
+        text context
+        int prompt_tokens
+        int completion_tokens
+        datetime created_at
+        int created_by FK
+        datetime updated_at
+        int updated_by FK
+        datetime last_used_at
+    }
+
+    PARTNER_RAG_RETRIEVER {
+        int partner_id FK
+        int tenant_id PK,FK
+        int retriever_id PK
+    }
+
+    LICENSE {
+        int id PK
+        int tenant_id FK
+        date start_date
+        date end_date
+    }
+
+    LICENSE_USAGE {
+        int id PK
+        int tenant_id FK
+    }
+
+    %% Relationships
+    TENANT ||--o{ USER : "has many"
+    TENANT ||--o{ TENANT_DOMAIN : "has many"
+    TENANT ||--o{ TENANT_PROJECT : "has many"
+    TENANT ||--o{ TENANT_MAKE : "has many"
+    TENANT ||--o| TENANT_MAKE : "default_tenant_make"
+    TENANT ||--o| PARTNER : "has one"
+    TENANT ||--o{ LICENSE : "has many"
+    TENANT ||--o{ LICENSE_USAGE : "has many"
+    TENANT ||--o{ SPECIALIST_MAGIC_LINK_TENANT : "has many"
+    TENANT ||--o{ PARTNER_TENANT : "has many"
+    TENANT ||--o{ PARTNER_RAG_RETRIEVER : "has many"
+
+    USER ||--o{ ROLES_USERS : "has many"
+
+    ROLE ||--o{ ROLES_USERS : "has many"
+
+    PARTNER ||--o{ PARTNER_SERVICE : "has many"
+    PARTNER ||--o{ PARTNER_RAG_RETRIEVER : "has many"
+
+    PARTNER_SERVICE ||--o{ PARTNER_TENANT : "has many"
+    PARTNER_SERVICE ||--o{ PARTNER_SERVICE_LICENSE_TIER : "has many"

eveai_app/templates/entitlements/edit_license_period.html

@@ -0,0 +1,24 @@
+{% extends 'base.html' %}
+{% from "macros.html" import render_field %}
+
+{% block title %}Edit Tenant Project{% endblock %}
+
+{% block content_title %}Edit License Period{% endblock %}
+{% block content_description %}Edit a License Period{% endblock %}
+
+{% block content %}
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {% set disabled_fields = [] %}
+        {% set exclude_fields = [] %}
+        <!-- Render Static Fields -->
+        {% for field in form %}
+            {{ render_field(field, disabled_fields, exclude_fields) }}
+        {% endfor %}
+        <button type="submit" class="btn btn-primary">Save Tenant Project</button>
+    </form>
+{% endblock %}
+
+{% block content_footer %}
+
+{% endblock %}

eveai_app/templates/navbar.html

@@ -72,6 +72,7 @@
                                     {'name': 'Tenants', 'url': '/user/tenants', 'roles': ['Super User', 'Partner Admin']},
                                     {'name': 'Tenant Overview', 'url': '/user/tenant_overview', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']},
                                     {'name': 'Edit Tenant', 'url': '/user/tenant/' ~ session['tenant'].get('id'), 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']},
+                                    {'name': 'Tenant Partner Services', 'url': '/user/tenant_partner_services', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']},
                                     {'name': 'Tenant Makes', 'url': '/user/tenant_makes', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']},
                                     {'name': 'Tenant Projects', 'url': '/user/tenant_projects', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']},
                                     {'name': 'Users', 'url': '/user/view_users', 'roles': ['Super User', 'Partner Admin', 'Tenant Admin']},

eveai_app/views/document_views.py

@@ -455,10 +455,10 @@ def add_url():
 
         except EveAIException as e:
             current_app.logger.error(f"Error adding document: {str(e)}")
-            flash(str(e), 'error')
+            flash(str(e), 'danger')
         except Exception as e:
             current_app.logger.error(f'Error adding document: {str(e)}')
-            flash('An error occurred while adding the document.', 'error')
+            flash('An error occurred while adding the document.', 'danger')
 
     return render_template('document/add_url.html', form=form)
 

eveai_app/views/entitlements_forms.py

@@ -5,6 +5,8 @@ from wtforms import (StringField, PasswordField, BooleanField, SubmitField, Emai
 from wtforms.validators import DataRequired, Length, Email, NumberRange, Optional, ValidationError, InputRequired
 import pytz
 
+from common.models.entitlements import PeriodStatus
+
 
 class LicenseTierForm(FlaskForm):
     name = StringField('Name', validators=[DataRequired(), Length(max=50)])
@@ -76,3 +78,14 @@ class LicenseForm(FlaskForm):
                                      validators=[DataRequired(), NumberRange(min=0)],
                                      default=0)
 
+
+class EditPeriodForm(FlaskForm):
+    period_start = DateField('Period Start', id='form-control datepicker', validators=[DataRequired()])
+    period_end = DateField('Period End', id='form-control datepicker', validators=[DataRequired()])
+    status = SelectField('Status', choices=[], validators=[DataRequired()])
+
+    def __init__(self, *args, **kwargs):
+        super(EditPeriodForm, self).__init__(*args, **kwargs)
+        self.status.choices = [(item.name, item.value) for item in PeriodStatus]
+
+

eveai_app/views/entitlements_views.py

@@ -13,11 +13,11 @@ from common.services.user import PartnerServices
 from common.services.user import UserServices
 from common.utils.eveai_exceptions import EveAIException
 from common.utils.security_utils import current_user_has_role
-from .entitlements_forms import LicenseTierForm, LicenseForm
+from .entitlements_forms import LicenseTierForm, LicenseForm, EditPeriodForm
 from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
 from common.utils.nginx_utils import prefixed_url_for
 from common.utils.document_utils import set_logging_information, update_logging_information
-from .list_views.entitlement_list_views import get_license_tiers_list_view, get_license_list_view
+from .list_views.entitlement_list_views import get_license_tiers_list_view, get_license_list_view, get_license_periods_list_view
 from .list_views.list_view_utils import render_list_view
 
 entitlements_bp = Blueprint('entitlements_bp', __name__, url_prefix='/entitlements')
@@ -255,14 +255,14 @@ def handle_license_selection():
         case 'edit_license':
             return redirect(prefixed_url_for('entitlements_bp.edit_license', license_id=license_id))
         case 'view_periods':
-            return redirect(prefixed_url_for('entitlements_bp.view_license_periods', license_id=license_id))
+            return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id))
         case _:
             return redirect(prefixed_url_for('entitlements_bp.licenses'))
 
 
 @entitlements_bp.route('/license/<int:license_id>/periods')
 @roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
-def view_license_periods(license_id):
+def license_periods(license_id):
     license = License.query.get_or_404(license_id)
 
     # Verify user can access this license
@@ -272,48 +272,77 @@ def view_license_periods(license_id):
             flash('Access denied to this license', 'danger')
             return redirect(prefixed_url_for('entitlements_bp.licenses'))
 
-    # Get all periods for this license
-    periods = (LicensePeriod.query
-               .filter_by(license_id=license_id)
-               .order_by(LicensePeriod.period_number)
-               .all())
+    config = get_license_periods_list_view(license_id)
 
-    # Group related data for easy template access
-    usage_by_period = {}
-    payments_by_period = {}
-    invoices_by_period = {}
+    # Check if there was an error in getting the configuration
+    if config.get('error'):
+        return render_template("index.html")
 
-    for period in periods:
-        usage_by_period[period.id] = period.license_usage
-        payments_by_period[period.id] = list(period.payments)
-        invoices_by_period[period.id] = list(period.invoices)
-
-    return render_template('entitlements/license_periods.html',
-                           license=license,
-                           periods=periods,
-                           usage_by_period=usage_by_period,
-                           payments_by_period=payments_by_period,
-                           invoices_by_period=invoices_by_period)
+    return render_list_view('list_view.html', **config)
 
 
-@entitlements_bp.route('/license/<int:license_id>/periods/<int:period_id>/transition', methods=['POST'])
-@roles_accepted('Super User', 'Partner Admin')
-def transition_period_status(license_id, period_id):
+@entitlements_bp.route('/license_period/<int:period_id>', methods=['GET', 'POST'])
+@roles_accepted('Super User')
+def edit_license_period(period_id):
     """Handle status transitions for license periods"""
     period = LicensePeriod.query.get_or_404(period_id)
-    new_status = request.form.get('new_status')
+    form = EditPeriodForm(obj=period)
 
-    try:
-        period.transition_status(PeriodStatus[new_status], current_user.id)
-        db.session.commit()
-        flash(f'Period {period.period_number} status updated to {new_status}', 'success')
-    except ValueError as e:
-        flash(f'Invalid status transition: {str(e)}', 'danger')
-    except Exception as e:
-        db.session.rollback()
-        flash(f'Error updating status: {str(e)}', 'danger')
+    if request.method == 'POST' and form.validate_on_submit():
+        form.populate_obj(period)
+        update_logging_information(period, dt.now(tz.utc))
+        match form.status.data:
+            case 'UPCOMING':
+                period.upcoming_at = dt.now(tz.utc)
+            case 'PENDING':
+                period.pending_at = dt.now(tz.utc)
+            case 'ACTIVE':
+                period.active_at = dt.now(tz.utc)
+            case 'COMPLETED':
+                period.completed_at = dt.now(tz.utc)
+            case 'INVOICED':
+                period.invoiced_at = dt.now(tz.utc)
+            case 'CLOSED':
+                period.closed_at = dt.now(tz.utc)
 
-    return redirect(prefixed_url_for('entitlements_bp.view_license_periods', license_id=license_id))
+        try:
+            db.session.add(period)
+            db.session.commit()
+            flash('Period updated successfully.', 'success')
+            current_app.logger.info(f"Successfully updated period {period_id}")
+        except SQLAlchemyError as e:
+            db.session.rollback()
+            flash(f'Error updating status: {str(e)}', 'danger')
+            current_app.logger.error(f"Error updating period {period_id}: {str(e)}")
+
+        return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=period.license_id))
+
+    return render_template('entitlements/edit_license_period.html', form=form)
+
+
+@entitlements_bp.route('/license/<int:license_id>/handle_period_selection', methods=['POST'])
+@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
+def handle_license_period_selection(license_id):
+    """Handle actions for license periods"""
+    action = request.form['action']
+
+    # For actions that don't require a selection
+    if 'selected_row' not in request.form:
+        return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id))
+
+    period_identification = request.form['selected_row']
+    period_id = ast.literal_eval(period_identification).get('value')
+
+    match action:
+        case 'view_period_details':
+            # TODO: Implement period details view if needed
+            flash('Period details view not yet implemented', 'info')
+            return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id))
+        case 'edit_license_period':
+            # Display a form to choose the new status
+            return redirect(prefixed_url_for('entitlements_bp.edit_license_period', period_id=period_id))
+        case _:
+            return redirect(prefixed_url_for('entitlements_bp.license_periods', license_id=license_id))
 
 
 @entitlements_bp.route('/view_licenses')

eveai_app/views/list_views/entitlement_list_views.py

@@ -3,7 +3,7 @@ from datetime import datetime as dt, timezone as tz
 from flask import flash
 from sqlalchemy import or_, desc
 
-from common.models.entitlements import LicenseTier, License
+from common.models.entitlements import LicenseTier, License, LicensePeriod
 from common.services.user import PartnerServices, UserServices
 from common.utils.eveai_exceptions import EveAIException
 from common.utils.security_utils import current_user_has_role
@@ -77,10 +77,16 @@ def get_license_tiers_list_view():
          'requiresSelection': False}
     ]
 
-    # Add assign license action if user has permission
+    # Add assign license actions if user has permission
+    current_app.logger.debug(f"Adding specific buttons")
     if UserServices.can_user_assign_license():
-        actions.insert(1, {'value': 'assign_license', 'text': 'Assign License', 'class': 'btn-info',
-                           'requiresSelection': True})
+        current_app.logger.debug(f"Adding Create License for Tenant")
+        actions.insert(1, {'value': 'create_license_for_tenant', 'text': 'Create License for Tenant',
+                           'class': 'btn-secondary', 'requiresSelection': True})
+    if current_user_has_role('Super User'):
+        current_app.logger.debug(f"Adding Associate License Tier to Partner")
+        actions.insert(2, {'value': 'associate_license_tier_to_partner', 'text': 'Associate License Tier to Partner',
+                           'class': 'btn-secondary','requiresSelection': True})
 
     # Initial sort configuration
     initial_sort = [{'column': 'start_date', 'dir': 'desc'}, {'column': 'id', 'dir': 'asc'}]
@@ -139,7 +145,7 @@ def get_license_list_view():
         {'title': 'License Tier', 'field': 'license_tier_name'},
         {'title': 'Start Date', 'field': 'start_date', 'width': 120},
         {'title': 'Nr of Periods', 'field': 'nr_of_periods', 'width': 120},
-        {'title': 'Active', 'field': 'active', 'formatter': 'tickCross', 'width': 100}
+        {'title': 'Active', 'field': 'active', 'formatter': 'tickCross', 'width': 80}
     ]
 
     # Action definitions
@@ -162,3 +168,74 @@ def get_license_list_view():
         'description': 'View and manage licenses',
         'table_height': 700
     }
+
+
+def get_license_periods_list_view(license_id):
+    """Generate the license periods list view configuration"""
+    # Get the license object
+    license = License.query.get_or_404(license_id)
+
+    # Get all periods for this license
+    periods = (LicensePeriod.query
+              .filter_by(license_id=license_id)
+              .order_by(LicensePeriod.period_number)
+              .all())
+
+    # Prepare data for Tabulator
+    data = []
+    for period in periods:
+        # Get usage data
+        usage = period.license_usage
+        storage_used = usage.storage_mb_used if usage else 0
+        embedding_used = usage.embedding_mb_used if usage else 0
+        interaction_used = usage.interaction_total_tokens_used if usage else 0
+
+        # Get payment status
+        prepaid_payment = period.prepaid_payment
+        prepaid_status = prepaid_payment.status.name if prepaid_payment else 'N/A'
+
+        # Get invoice status
+        prepaid_invoice = period.prepaid_invoice
+        invoice_status = prepaid_invoice.status.name if prepaid_invoice else 'N/A'
+
+        data.append({
+            'id': period.id,
+            'period_number': period.period_number,
+            'period_start': period.period_start.strftime('%Y-%m-%d'),
+            'period_end': period.period_end.strftime('%Y-%m-%d'),
+            'status': period.status.name,
+        })
+
+    # Column definitions
+    columns = [
+        {'title': 'ID', 'field': 'id', 'width': 60, 'type': 'number'},
+        {'title': 'Period', 'field': 'period_number'},
+        {'title': 'Start Date', 'field': 'period_start'},
+        {'title': 'End Date', 'field': 'period_end'},
+        {'title': 'Status', 'field': 'status'},
+    ]
+
+    # Action definitions
+    actions = [
+        {'value': 'view_period_details', 'text': 'View Details', 'class': 'btn-primary', 'requiresSelection': True}
+    ]
+
+    # If user has admin roles, add transition action
+    if current_user_has_role('Super User'):
+        actions.append({'value': 'edit_license_period', 'text': 'Edit Period', 'class': 'btn-secondary', 'requiresSelection': True})
+
+    # Initial sort configuration
+    initial_sort = [{'column': 'period_number', 'dir': 'asc'}]
+
+    return {
+        'title': 'License Periods',
+        'data': data,
+        'columns': columns,
+        'actions': actions,
+        'initial_sort': initial_sort,
+        'table_id': 'license_periods_table',
+        'form_action': url_for('entitlements_bp.handle_license_period_selection', license_id=license_id),
+        'description': f'View and manage periods for License {license_id}',
+        'table_height': 700,
+        'license': license
+    }

eveai_app/views/list_views/partner_list_views.py

@@ -10,9 +10,7 @@ def get_partners_list_view():
     # Haal alle partners op met hun tenant informatie
     query = (db.session.query(
         Partner.id,
-        Partner.code,
         Partner.active,
-        Partner.logo_url,
         Tenant.name.label('name')
     ).join(Tenant, Partner.tenant_id == Tenant.id).order_by(Partner.id))
 
@@ -24,7 +22,6 @@ def get_partners_list_view():
         for partner in all_partners:
             data.append({
                 'id': partner.id,
-                'code': partner.code,
                 'name': partner.name,
                 'active': partner.active
             })
@@ -32,7 +29,6 @@ def get_partners_list_view():
         # Kolomdefinities
         columns = [
             {'title': 'ID', 'field': 'id', 'width': 80},
-            {'title': 'Code', 'field': 'code'},
             {'title': 'Name', 'field': 'name'},
             {'title': 'Active', 'field': 'active', 'formatter': 'tickCross'}
         ]

eveai_app/views/list_views/user_list_views.py

@@ -3,7 +3,7 @@ from flask_security import roles_accepted
 from sqlalchemy.exc import SQLAlchemyError
 import ast
 
-from common.models.user import Tenant, User, TenantDomain, TenantProject, TenantMake
+from common.models.user import Tenant, User, TenantDomain, TenantProject, TenantMake, PartnerTenant, PartnerService
 from common.services.user import UserServices
 from eveai_app.views.list_views.list_view_utils import render_list_view
 
@@ -232,3 +232,47 @@ def get_tenant_makes_list_view(tenant_id):
         'form_action': url_for('user_bp.handle_tenant_make_selection'),
         'description': f'Makes for tenant {tenant_id}'
     }
+
+
+    # Tenant Partner Services list view helper
+def get_tenant_partner_services_list_view(tenant_id):
+    """Generate the tenant partner services list view configuration for a specific tenant"""
+    # Get partner services for the tenant through PartnerTenant association
+    query = PartnerService.query.join(PartnerTenant).filter(PartnerTenant.tenant_id == tenant_id)
+    partner_services = query.all()
+
+    # Prepare data for Tabulator
+    data = []
+    for service in partner_services:
+        data.append({
+            'id': service.id,
+            'name': service.name,
+            'type': service.type,
+            'type_version': service.type_version,
+            'active': service.active
+        })
+
+    # Column Definitions
+    columns = [
+        {'title': 'ID', 'field': 'id', 'width': 80},
+        {'title': 'Name', 'field': 'name'},
+        {'title': 'Type', 'field': 'type'},
+        {'title': 'Version', 'field': 'type_version'},
+        {'title': 'Active', 'field': 'active', 'formatter': 'tickCross', 'width': 120}
+    ]
+
+    # No actions needed as specified in requirements
+    actions = []
+
+    initial_sort = [{'column': 'name', 'dir': 'asc'}]
+
+    return {
+        'title': 'Partner Services',
+        'data': data,
+        'columns': columns,
+        'actions': actions,
+        'initial_sort': initial_sort,
+        'table_id': 'tenant_partner_services_table',
+        'form_action': url_for('user_bp.tenant_partner_services'),
+        'description': f'Partner Services for tenant {tenant_id}'
+    }

eveai_app/views/user_views.py

@@ -24,7 +24,8 @@ from common.services.user import UserServices
 from common.utils.mail_utils import send_email
 
 from eveai_app.views.list_views.user_list_views import get_tenants_list_view, get_users_list_view, \
-    get_tenant_domains_list_view, get_tenant_projects_list_view, get_tenant_makes_list_view
+    get_tenant_domains_list_view, get_tenant_projects_list_view, get_tenant_makes_list_view, \
+    get_tenant_partner_services_list_view
 from eveai_app.views.list_views.list_view_utils import render_list_view
 
 user_bp = Blueprint('user_bp', __name__, url_prefix='/user')
@@ -686,6 +687,15 @@ def handle_tenant_make_selection():
     return redirect(prefixed_url_for('user_bp.tenant_makes'))
 
 
+@user_bp.route('/tenant_partner_services', methods=['GET', 'POST'])
+@roles_accepted('Super User', 'Partner Admin', 'Tenant Admin')
+def tenant_partner_services():
+    tenant_id = session['tenant']['id']
+    config = get_tenant_partner_services_list_view(tenant_id)
+    return render_list_view('list_view.html', **config)
+
+
+
 def reset_uniquifier(user):
     security.datastore.set_uniquifier(user)
     db.session.add(user)

eveai_chat_workers/retrievers/globals/PARTNER_RAG/1_0.py

@@ -0,0 +1,148 @@
+# retrievers/standard_rag.py
+import json
+from datetime import datetime as dt, timezone as tz
+from typing import Dict, Any, List
+from sqlalchemy import func, or_, desc, text
+from sqlalchemy.exc import SQLAlchemyError
+from flask import current_app
+
+from common.extensions import db
+from flask_sqlalchemy import SQLAlchemy
+from common.models.document import Document, DocumentVersion, Catalog, Retriever
+from common.models.user import Tenant, PartnerRAGRetriever
+from common.utils.datetime_utils import get_date_in_timezone
+from common.utils.model_utils import get_embedding_model_and_class
+from eveai_chat_workers.retrievers.base_retriever import BaseRetriever
+
+from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments, RetrieverResult, RetrieverMetadata
+
+
+class RetrieverExecutor(BaseRetriever):
+    """Standard RAG retriever implementation"""
+
+    def __init__(self, tenant_id: int, retriever_id: int, partner_id: int):
+        super().__init__(tenant_id, retriever_id)
+        self.partner_id = partner_id
+        self.partner_db = SQLAlchemy()
+        self._init_partner_db()
+
+    def _init_partner_db(self):
+        self.partner_db.init_app(current_app)
+        self.partner_db.session.execute(text(f'set search_path to "{str(self.tenant_id)}", public'))
+        self.partner_db.session.commit()
+
+    @property
+    def type(self) -> str:
+        return "PARTNER_RAG"
+
+    @property
+    def type_version(self) -> str:
+        return "1.0"
+
+    def retrieve(self, arguments: RetrieverArguments) -> List[RetrieverResult]:
+        """
+        Retrieve documents based on query
+
+        Args:
+            arguments: Validated RetrieverArguments containing at minimum:
+                - query: str - The search query
+
+        Returns:
+            List[RetrieverResult]: List of retrieved documents with similarity scores
+        """
+        try:
+            question = arguments.question
+
+            # Get query embedding
+            query_embedding = self.embedding_model.embed_query(question)
+
+            # Get the appropriate embedding database model
+            db_class = self.embedding_model_class
+
+            # Get the current date for validity checks
+            current_date = dt.now(tz=tz.utc).date()
+
+            # Create subquery for latest versions
+            subquery = (
+                self.partner_db.session.query(
+                    DocumentVersion.doc_id,
+                    func.max(DocumentVersion.id).label('latest_version_id')
+                )
+                .group_by(DocumentVersion.doc_id)
+                .subquery()
+            )
+
+            similarity_threshold = self.retriever.configuration.get('es_similarity_threshold', 0.3)
+            k = self.retriever.configuration.get('es_k', 8)
+
+            # Main query
+            query_obj = (
+                self.partner_db.session.query(
+                    db_class,
+                    DocumentVersion.url,
+                    (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity')
+                )
+                .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id)
+                .join(Document, DocumentVersion.doc_id == Document.id)
+                .join(subquery, DocumentVersion.id == subquery.c.latest_version_id)
+                .filter(
+                    or_(Document.valid_from.is_(None), func.date(Document.valid_from) <= current_date),
+                    or_(Document.valid_to.is_(None), func.date(Document.valid_to) >= current_date),
+                    (1 - db_class.embedding.cosine_distance(query_embedding)) > similarity_threshold,
+                    Document.catalog_id == self.catalog_id
+                )
+                .order_by(desc('similarity'))
+                .limit(k)
+            )
+
+            results = query_obj.all()
+
+            # Transform results into standard format
+            processed_results = []
+            for doc, url, similarity in results:
+                # Parse user_metadata to ensure it's a dictionary
+                user_metadata = self._parse_metadata(doc.document_version.user_metadata)
+                processed_results.append(
+                    RetrieverResult(
+                        id=doc.id,
+                        chunk=doc.chunk,
+                        similarity=float(similarity),
+                        metadata=RetrieverMetadata(
+                            document_id=doc.document_version.doc_id,
+                            version_id=doc.document_version.id,
+                            document_name=f"Partner {self.partner_id} with Tenant ID {self.tenant_id} Doc: "
+                                          f"{doc.document_version.document.name}",
+                            url=url or "",
+                            user_metadata=user_metadata,
+                        )
+                    )
+                )
+
+            # Log the retrieval
+            if self.tuning:
+                compiled_query = str(query_obj.statement.compile(
+                    compile_kwargs={"literal_binds": True}  # This will include the actual values in the SQL
+                ))
+                self.log_tuning('retrieve', {
+                    "arguments": arguments.model_dump(),
+                    "similarity_threshold": similarity_threshold,
+                    "k": k,
+                    "query": compiled_query,
+                    "Raw Results": str(results),
+                    "Processed Results": [r.model_dump() for r in processed_results],
+                })
+
+            return processed_results
+
+        except SQLAlchemyError as e:
+            current_app.logger.error(f'Error in RAG retrieval: {e}')
+            self.partner_db.session.rollback()
+            raise
+        except Exception as e:
+            current_app.logger.error(f'Unexpected error in RAG retrieval: {e}')
+            raise
+        finally:
+            if self.partner_db.session:
+                self.partner_db.session.close()
+
+

eveai_chat_workers/specialists/evie_partner/PARTNER_RAG_SPECIALIST/1_0.py

@@ -0,0 +1,243 @@
+import json
+from os import wait
+from typing import Optional, List, Dict, Any
+
+from crewai.flow.flow import start, listen, and_
+from flask import current_app
+from pydantic import BaseModel, Field
+
+from common.extensions import db
+from common.models.interaction import SpecialistRetriever
+from common.models.user import Partner, PartnerService, PartnerTenant, PartnerRAGRetriever
+from common.services.utils.translation_services import TranslationServices
+from common.utils.business_event_context import current_event
+from eveai_chat_workers.retrievers.base_retriever import BaseRetriever, get_retriever_class
+from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments
+from eveai_chat_workers.specialists.crewai_base_specialist import CrewAIBaseSpecialistExecutor
+from eveai_chat_workers.specialists.specialist_typing import SpecialistResult, SpecialistArguments
+from eveai_chat_workers.outputs.globals.rag.rag_v1_0 import RAGOutput
+from eveai_chat_workers.specialists.crewai_base_classes import EveAICrewAICrew, EveAICrewAIFlow, EveAIFlowState
+
+INSUFFICIENT_INFORMATION_MESSAGE = (
+    "We do not have the necessary information to provide you with the requested answers. "
+    "Please accept our apologies. Don't hesitate to ask other questions, and I'll do my best to answer them.")
+
+
+class SpecialistExecutor(CrewAIBaseSpecialistExecutor):
+    """
+    type: RAG_SPECIALIST
+    type_version: 1.0
+    RAG Specialist Executor class
+    """
+
+    def __init__(self, tenant_id, specialist_id, session_id, task_id, **kwargs):
+        self.rag_crew = None
+
+        super().__init__(tenant_id, specialist_id, session_id, task_id)
+
+    @property
+    def type(self) -> str:
+        return "RAG_SPECIALIST"
+
+    @property
+    def type_version(self) -> str:
+        return "1.1"
+
+    def _initialize_retrievers(self) -> List[BaseRetriever]:
+        """Initialize all retrievers associated with this specialist"""
+        retrievers = []
+
+        partner_ids = (
+            db.session.query(Partner.id)
+            .join(PartnerService, Partner.id == PartnerService.partner_id)
+            .join(PartnerTenant, PartnerService.id == PartnerTenant.partner_service_id)
+            .filter(PartnerTenant.tenant_id == self.tenant_id)
+            .distinct()
+            .all()
+        )
+
+        # Extract the actual partner IDs from the query result
+        partner_ids_list = [partner_id[0] for partner_id in partner_ids]
+
+        # Get all corresponding PartnerRagRetrievers for the partner_ids list
+        partner_rag_retrievers = (
+            PartnerRAGRetriever.query
+            .filter(PartnerRAGRetriever.partner_id.in_(partner_ids_list))
+            .filter(PartnerRAGRetriever.tenant_id == self.tenant_id)
+            .all()
+        )
+
+        retriever_executor_class = get_retriever_class("PARTNER_RAG", "1.0")
+        # Get retriever associations from database
+
+        self.log_tuning("_initialize_retrievers", {"Nr of partner retrievers": len(partner_rag_retrievers)})
+
+        for partner_rag_retriever in partner_rag_retrievers :
+            # Get retriever configuration from database
+            self.log_tuning("_initialize_retrievers", {
+                "Partner id": partner_rag_retriever.partner_id,
+                "Tenant id": partner_rag_retriever.tenant_id,
+                "Retriever id": partner_rag_retriever.retriever_id,
+            })
+
+            retriever_executor = retriever_executor_class(partner_rag_retriever.tenant_id,
+                                                          partner_rag_retriever.retriever_id)
+            # Initialize retriever with its configuration
+            retrievers.append(retriever_executor)
+
+        return retrievers
+
+    def _config_task_agents(self):
+        self._add_task_agent("rag_task", "rag_agent")
+
+    def _config_pydantic_outputs(self):
+        self._add_pydantic_output("rag_task", RAGOutput, "rag_output")
+
+    def _config_state_result_relations(self):
+        self._add_state_result_relation("rag_output")
+        self._add_state_result_relation("citations")
+
+    def _instantiate_specialist(self):
+        verbose = self.tuning
+
+        rag_agents = [self.rag_agent]
+        rag_tasks = [self.rag_task]
+        self.rag_crew = EveAICrewAICrew(
+            self,
+            "Rag Crew",
+            agents=rag_agents,
+            tasks=rag_tasks,
+            verbose=verbose,
+        )
+
+        self.flow = RAGFlow(
+            self,
+            self.rag_crew,
+        )
+
+    def execute(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("RAG Specialist execution started", {})
+
+        if not self._cached_session.interactions:
+            specialist_phase = "initial"
+        else:
+            specialist_phase = self._cached_session.interactions[-1].specialist_results.get('phase', 'initial')
+
+        results = None
+
+        match specialist_phase:
+            case "initial":
+                results = self.execute_initial_state(arguments, formatted_context, citations)
+            case "rag":
+                results = self.execute_rag_state(arguments, formatted_context, citations)
+
+        self.log_tuning(f"RAG Specialist execution ended", {"Results": results.model_dump()})
+
+        return results
+
+    def execute_initial_state(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("RAG Specialist initial_state execution started", {})
+
+        welcome_message = self.specialist.configuration.get('welcome_message', 'Welcome! You can start asking questions')
+        welcome_message = TranslationServices.translate(self.tenant_id, welcome_message, arguments.language)
+
+        self.flow.state.answer = welcome_message
+        self.flow.state.phase = "rag"
+
+        results = RAGSpecialistResult.create_for_type(self.type, self.type_version)
+
+        return results
+
+    def execute_rag_state(self, arguments: SpecialistArguments, formatted_context, citations) -> SpecialistResult:
+        self.log_tuning("RAG Specialist rag_state execution started", {})
+
+        insufficient_info_message = TranslationServices.translate(self.tenant_id,
+                                                                  INSUFFICIENT_INFORMATION_MESSAGE,
+                                                                  arguments.language)
+
+        formatted_context, citations = self._retrieve_context(arguments)
+
+        if formatted_context:
+            flow_inputs = {
+                "language": arguments.language,
+                "question": arguments.question,
+                "context": formatted_context,
+                "history": self.formatted_history,
+                "name": self.specialist.configuration.get('name', ''),
+                "welcome_message": self.specialist.configuration.get('welcome_message', '')
+            }
+
+            flow_results = self.flow.kickoff(inputs=flow_inputs)
+
+            if flow_results.rag_output.insufficient_info:
+                flow_results.rag_output.answer = insufficient_info_message
+
+            rag_output = flow_results.rag_output
+        else:
+            rag_output = RAGOutput(answer=insufficient_info_message, insufficient_info=True)
+
+        self.flow.state.rag_output = rag_output
+        self.flow.state.citations = citations
+        self.flow.state.answer = rag_output.answer
+        self.flow.state.phase = "rag"
+
+        results = RAGSpecialistResult.create_for_type(self.type, self.type_version)
+
+        return results
+
+
+class RAGSpecialistInput(BaseModel):
+    language: Optional[str] = Field(None, alias="language")
+    question: Optional[str] = Field(None, alias="question")
+    context: Optional[str] = Field(None, alias="context")
+    history: Optional[str] = Field(None, alias="history")
+    name: Optional[str] = Field(None, alias="name")
+    welcome_message: Optional[str] = Field(None, alias="welcome_message")
+
+
+class RAGSpecialistResult(SpecialistResult):
+    rag_output: Optional[RAGOutput] = Field(None, alias="Rag Output")
+
+
+class RAGFlowState(EveAIFlowState):
+    """Flow state for RAG specialist that automatically updates from task outputs"""
+    input: Optional[RAGSpecialistInput] = None
+    rag_output: Optional[RAGOutput] = None
+    citations: Optional[List[Dict[str, Any]]] = None
+
+
+class RAGFlow(EveAICrewAIFlow[RAGFlowState]):
+    def __init__(self,
+                 specialist_executor: CrewAIBaseSpecialistExecutor,
+                 rag_crew: EveAICrewAICrew,
+                 **kwargs):
+        super().__init__(specialist_executor, "RAG Specialist Flow", **kwargs)
+        self.specialist_executor = specialist_executor
+        self.rag_crew = rag_crew
+        self.exception_raised = False
+
+    @start()
+    def process_inputs(self):
+        return ""
+
+    @listen(process_inputs)
+    async def execute_rag(self):
+        inputs = self.state.input.model_dump()
+        try:
+            crew_output = await self.rag_crew.kickoff_async(inputs=inputs)
+            self.specialist_executor.log_tuning("RAG Crew Output", crew_output.model_dump())
+            output_pydantic = crew_output.pydantic
+            if not output_pydantic:
+                raw_json = json.loads(crew_output.raw)
+                output_pydantic = RAGOutput.model_validate(raw_json)
+            self.state.rag_output = output_pydantic
+            return crew_output
+        except Exception as e:
+            current_app.logger.error(f"CREW rag_crew Kickoff Error: {str(e)}")
+            self.exception_raised = True
+            raise e
+
+    async def kickoff_async(self, inputs=None):
+        self.state.input = RAGSpecialistInput.model_validate(inputs)
+        result = await super().kickoff_async(inputs)
+        return self.state

eveai_chat_workers/specialists/globals/STANDARD_RAG_SPECIALIST/1_0.py

@@ -1,239 +0,0 @@
-from datetime import datetime
-from typing import Dict, Any, List
-from flask import current_app
-from langchain_core.exceptions import LangChainException
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnableParallel, RunnablePassthrough
-
-from common.langchain.outputs.base import OutputRegistry
-from common.langchain.outputs.rag import RAGOutput
-from common.utils.business_event_context import current_event
-from eveai_chat_workers.specialists.specialist_typing import SpecialistArguments, SpecialistResult
-from eveai_chat_workers.chat_session_cache import get_chat_history
-from common.models.interaction import Specialist
-from common.utils.model_utils import create_language_template, replace_variable_in_template, \
-    get_template
-from eveai_chat_workers.specialists.base_specialist import BaseSpecialistExecutor
-from eveai_chat_workers.retrievers.retriever_typing import RetrieverArguments
-
-
-class SpecialistExecutor(BaseSpecialistExecutor):
-    """
-    type: STANDARD_RAG
-    type_version: 1.0
-    Standard Q&A RAG Specialist implementation that combines retriever results
-    with LLM processing to generate answers.
-    """
-
-    def __init__(self, tenant_id: int, specialist_id: int, session_id: str, task_id: str):
-        super().__init__(tenant_id, specialist_id, session_id, task_id)
-
-        # Check and load the specialist
-        specialist = Specialist.query.get_or_404(specialist_id)
-        # Set the specific configuration for the RAG Specialist
-        self.specialist_context = specialist.configuration.get('specialist_context', '')
-        self.temperature = specialist.configuration.get('temperature', 0.3)
-        self.tuning = specialist.tuning
-
-        # Initialize retrievers
-        self.retrievers = self._initialize_retrievers()
-
-    @property
-    def type(self) -> str:
-        return "STANDARD_RAG_SPECIALIST"
-
-    @property
-    def type_version(self) -> str:
-        return "1.0"
-
-    @property
-    def required_templates(self) -> List[str]:
-        """List of required templates for this specialist"""
-        return ['rag', 'history']
-
-    def _detail_question(self, language: str, question: str) -> str:
-        """Detail question based on conversation history"""
-        try:
-            # Get cached session history
-            cached_session = get_chat_history(self.session_id)
-
-            # Format history for the prompt
-            formatted_history = "\n\n".join([
-                f"HUMAN:\n{interaction.specialist_results.get('detailed_query')}\n\n"
-                f"AI:\n{interaction.specialist_results.get('answer')}"
-                for interaction in cached_session.interactions
-            ])
-
-            # Get LLM and template
-            template, llm = get_template("history", temperature=0.3)
-            language_template = create_language_template(template, language)
-
-            # Create prompt
-            history_prompt = ChatPromptTemplate.from_template(language_template)
-
-            # Create chain
-            chain = (
-                    history_prompt |
-                    llm |
-                    StrOutputParser()
-            )
-
-            # Execute chain
-            detailed_question = chain.invoke({
-                "history": formatted_history,
-                "question": question
-            })
-
-            if self.tuning:
-                self.log_tuning("_detail_question", {
-                    "cached_session_id": cached_session.session_id,
-                    "cached_session.interactions": str(cached_session.interactions),
-                    "original_question": question,
-                    "history_used": formatted_history,
-                    "detailed_question": detailed_question,
-                })
-
-            return detailed_question
-
-        except Exception as e:
-            current_app.logger.error(f"Error detailing question: {e}")
-            return question  # Fallback to original question
-
-    def execute(self, arguments: SpecialistArguments) -> SpecialistResult:
-        """
-        Execute the RAG specialist to generate an answer
-        """
-        start_time = datetime.now()
-
-        try:
-            with current_event.create_span("Specialist Detail Question"):
-                self.update_progress("Detail Question Start", {})
-                # Get required arguments
-                language = arguments.language
-                query = arguments.query
-                detailed_question = self._detail_question(language, query)
-                self.update_progress("Detail Question End", {})
-
-            # Log the start of retrieval process if tuning is enabled
-            with current_event.create_span("Specialist Retrieval"):
-                self.log_tuning("Starting context retrieval", {
-                    "num_retrievers": len(self.retrievers),
-                    "all arguments": arguments.model_dump(),
-                })
-                self.update_progress("EveAI Retriever Start", {})
-
-                # Get retriever-specific arguments
-                retriever_arguments = arguments.retriever_arguments
-
-                # Collect context from all retrievers
-                all_context = []
-                for retriever in self.retrievers:
-                    # Get arguments for this specific retriever
-                    retriever_id = str(retriever.retriever_id)
-                    if retriever_id not in retriever_arguments:
-                        current_app.logger.error(f"Missing arguments for retriever {retriever_id}")
-                        continue
-
-                    # Get the retriever's arguments and update the query
-                    current_retriever_args = retriever_arguments[retriever_id]
-                    if isinstance(retriever_arguments[retriever_id], RetrieverArguments):
-                        updated_args = current_retriever_args.model_dump()
-                        updated_args['query'] = detailed_question
-                        retriever_args = RetrieverArguments(**updated_args)
-                    else:
-                        # Create a new RetrieverArguments instance from the dictionary
-                        current_retriever_args['query'] = detailed_question
-                        retriever_args = RetrieverArguments(**current_retriever_args)
-
-                    # Each retriever gets its own specific arguments
-                    retriever_result = retriever.retrieve(retriever_args)
-                    all_context.extend(retriever_result)
-
-                # Sort by similarity if available and get unique contexts
-                all_context.sort(key=lambda x: x.similarity, reverse=True)
-                unique_contexts = []
-                seen_chunks = set()
-                for ctx in all_context:
-                    if ctx.chunk not in seen_chunks:
-                        unique_contexts.append(ctx)
-                        seen_chunks.add(ctx.chunk)
-
-                self.log_tuning("Context retrieval completed", {
-                    "total_contexts": len(all_context),
-                    "unique_contexts": len(unique_contexts),
-                    "average_similarity": sum(ctx.similarity for ctx in unique_contexts) / len(
-                        unique_contexts) if unique_contexts else 0
-                })
-                self.update_progress("EveAI Retriever Complete", {})
-
-            # Prepare context for LLM
-            formatted_context = "\n\n".join([
-                f"SOURCE: {ctx.metadata.document_id}\n\n{ctx.chunk}"
-                for ctx in unique_contexts
-            ])
-
-            with current_event.create_span("Specialist RAG invocation"):
-                try:
-                    self.update_progress(self.task_id, "EveAI Chain Start", {})
-                    template, llm = get_template("rag", self.temperature)
-                    language_template = create_language_template(template, language)
-                    full_template = replace_variable_in_template(
-                        language_template,
-                        "{tenant_context}",
-                        self.specialist_context
-                    )
-
-                    if self.tuning:
-                        self.log_tuning("Template preparation completed", {
-                            "template": full_template,
-                            "context": formatted_context,
-                            "tenant_context": self.specialist_context,
-                        })
-
-                    # Create prompt
-                    rag_prompt = ChatPromptTemplate.from_template(full_template)
-
-                    # Setup chain components
-                    setup_and_retrieval = RunnableParallel({
-                        "context": lambda x: formatted_context,
-                        "question": lambda x: x
-                    })
-
-                    # Get output schema for structured output
-                    output_schema = OutputRegistry.get_schema(self.type)
-                    structured_llm = llm.with_structured_output(output_schema)
-                    chain = setup_and_retrieval | rag_prompt | structured_llm
-
-                    raw_result = chain.invoke(detailed_question)
-                    result = SpecialistResult.create_for_type(
-                        self.type,
-                        self.type_version,
-                        detailed_question=detailed_question,
-                        answer=raw_result.answer,
-                        citations=[ctx.metadata.document_id for ctx in unique_contexts
-                                   if ctx.id in raw_result.citations],
-                        insufficient_info=raw_result.insufficient_info
-                    )
-
-                    if self.tuning:
-                        self.log_tuning("LLM chain execution completed", {
-                            "Result": result.model_dump()
-                        })
-                    self.update_progress("EveAI Chain Complete", {})
-
-                except Exception as e:
-                    current_app.logger.error(f"Error in LLM processing: {e}")
-                    if self.tuning:
-                        self.log_tuning("LLM processing error", {"error": str(e)})
-                    raise
-
-            return result
-
-        except Exception as e:
-            current_app.logger.error(f'Error in RAG specialist execution: {str(e)}')
-            raise
-
-
-# Register the specialist type
-OutputRegistry.register("STANDARD_RAG_SPECIALIST", RAGOutput)