From 9652d0bff9f1a2cab7ead5ed0d93e187c2636f58 Mon Sep 17 00:00:00 2001 From: Josako Date: Tue, 22 Apr 2025 13:49:38 +0200 Subject: [PATCH] - RAG & SPIN Specialist improvements --- .../tasks/IDENTIFICATION_DETECTION_TASK/1.0.0.yaml | 5 +---- .../tasks/IDENTIFICATION_QUESTIONS_TASK/1.0.0.yaml | 7 +------ config/tasks/RAG_CONSOLIDATION_TASK/1.0.0.yaml | 1 - config/tasks/RAG_TASK/1.0.0.yaml | 4 +--- config/tasks/SPIN_DETECT_TASK/1.0.0.yaml | 8 +------- config/tasks/SPIN_QUESTIONS_TASK/1.0.0.yaml | 12 +----------- eveai_chat_workers/outputs/rag/rag_v1_0.py | 6 +++--- eveai_chat_workers/retrievers/retriever_typing.py | 3 ++- eveai_chat_workers/retrievers/standard_rag.py | 4 +++- eveai_chat_workers/specialists/RAG_SPECIALIST/1_0.py | 1 - .../specialists/SPIN_SPECIALIST/1_0.py | 8 ++++++-- .../specialists/crewai_base_specialist.py | 6 +++++- 12 files changed, 24 insertions(+), 41 deletions(-) diff --git a/config/tasks/IDENTIFICATION_DETECTION_TASK/1.0.0.yaml b/config/tasks/IDENTIFICATION_DETECTION_TASK/1.0.0.yaml index b80a846..d8ca17d 100644 --- a/config/tasks/IDENTIFICATION_DETECTION_TASK/1.0.0.yaml +++ b/config/tasks/IDENTIFICATION_DETECTION_TASK/1.0.0.yaml @@ -16,10 +16,7 @@ task_description: > {custom_description} expected_output: > - - Personal Identification information such as name, email, phone number, job title, and any additional information that - may prove to be interesting in the current or future conversations. - - Company information such as company name, industry, size, company website, ... - {custom_expected_output} + metadata: author: "Josako" date_added: "2025-01-08" diff --git a/config/tasks/IDENTIFICATION_QUESTIONS_TASK/1.0.0.yaml b/config/tasks/IDENTIFICATION_QUESTIONS_TASK/1.0.0.yaml index ad35ad0..624c033 100644 --- a/config/tasks/IDENTIFICATION_QUESTIONS_TASK/1.0.0.yaml +++ b/config/tasks/IDENTIFICATION_QUESTIONS_TASK/1.0.0.yaml @@ -11,12 +11,7 @@ task_description: > {custom_description} expected_output: > - - Personal Identification information such as name, email, phone number, job title, and any additional information that - may prove to be interesting in the current or future conversations. - - Company information such as company name, industry, size, company website, ... - {custom_expected_output} - - Top {nr_of_questions} questions to ask in order to complete identification. - {custom_expected_output} + metadata: author: "Josako" date_added: "2025-01-08" diff --git a/config/tasks/RAG_CONSOLIDATION_TASK/1.0.0.yaml b/config/tasks/RAG_CONSOLIDATION_TASK/1.0.0.yaml index 66255d5..a70f1e1 100644 --- a/config/tasks/RAG_CONSOLIDATION_TASK/1.0.0.yaml +++ b/config/tasks/RAG_CONSOLIDATION_TASK/1.0.0.yaml @@ -19,7 +19,6 @@ task_description: > %%%{additional_questions}%%% expected_output: > - One consolidated communication towards the end user. {custom_expected_output} metadata: author: "Josako" diff --git a/config/tasks/RAG_TASK/1.0.0.yaml b/config/tasks/RAG_TASK/1.0.0.yaml index f8612d3..209d337 100644 --- a/config/tasks/RAG_TASK/1.0.0.yaml +++ b/config/tasks/RAG_TASK/1.0.0.yaml @@ -14,9 +14,7 @@ task_description: > Query: {query} expected_output: > - - Answer - - A list of sources used in generating the answer, citations - - An indication (True or False) if there's insufficient information to give an answer. + metadata: author: "Josako" date_added: "2025-01-08" diff --git a/config/tasks/SPIN_DETECT_TASK/1.0.0.yaml b/config/tasks/SPIN_DETECT_TASK/1.0.0.yaml index 64e13c9..f496764 100644 --- a/config/tasks/SPIN_DETECT_TASK/1.0.0.yaml +++ b/config/tasks/SPIN_DETECT_TASK/1.0.0.yaml @@ -10,13 +10,7 @@ task_description: > Latest reply: {query} expected_output: > - The SPIN analysis, comprised of: - - Situation information: a description of the customer's current context / situation. - - Problem information: a description of the customer's problems uncovering it's challenges and pain points. - - Implication information: implications of situation / identified problems, i.e. of the consequences of those problems. - - Need-payoff information: Customer's needs, helping customers realize value of solutions. - - Additional info: Information that does not fit in the above SPIN-categories, but that can be commercially - interesting, including if provided: {custom_expected_output} + metadata: author: "Josako" date_added: "2025-01-08" diff --git a/config/tasks/SPIN_QUESTIONS_TASK/1.0.0.yaml b/config/tasks/SPIN_QUESTIONS_TASK/1.0.0.yaml index b35ea1b..87f722f 100644 --- a/config/tasks/SPIN_QUESTIONS_TASK/1.0.0.yaml +++ b/config/tasks/SPIN_QUESTIONS_TASK/1.0.0.yaml @@ -12,17 +12,7 @@ task_description: > Latest reply: {query} expected_output: > - The SPIN analysis, comprised of: - - Situation information: a description of the customer's current context / situation. - - Problem information: a description of the customer's problems uncovering it's challenges and pain points. - - Implication information: implications of situation / identified problems, i.e. of the consequences of those problems. - - Need-payoff information: Customer's needs, helping customers realize value of solutions. - - Additional info: Information that does not fit in the above SPIN-categories, but that can be commercially interesting. - The SPIN questions: - - At max {nr_of_questions} questions to complete the SPIN-context of the customer, as a markdown list without '```'. - Potential Customer Indication: - - An indication if - given the current SPIN - this could be a good customer (True) or not (False). - {custom_expected_output} + metadata: author: "Josako" date_added: "2025-01-08" diff --git a/eveai_chat_workers/outputs/rag/rag_v1_0.py b/eveai_chat_workers/outputs/rag/rag_v1_0.py index 9930fe0..23d22a9 100644 --- a/eveai_chat_workers/outputs/rag/rag_v1_0.py +++ b/eveai_chat_workers/outputs/rag/rag_v1_0.py @@ -4,6 +4,6 @@ from pydantic import BaseModel, Field class RAGOutput(BaseModel): - answer: Optional[str] = Field(None, description="Answer to the questions asked") - citations: Optional[List[str]] = Field(None, description="A list of sources used in generating the answer") - insufficient_info: Optional[bool] = Field(None, description="An indication if there's insufficient information to answer") + answer: str = Field(None, description="Answer to the questions asked") + insufficient_info: bool = Field(None, description="An indication if there's insufficient information to answer") + diff --git a/eveai_chat_workers/retrievers/retriever_typing.py b/eveai_chat_workers/retrievers/retriever_typing.py index 8cc54b1..eda575c 100644 --- a/eveai_chat_workers/retrievers/retriever_typing.py +++ b/eveai_chat_workers/retrievers/retriever_typing.py @@ -1,4 +1,4 @@ -from typing import Dict, Any +from typing import Dict, Any, Optional from flask import current_app from pydantic import BaseModel, Field, model_validator @@ -11,6 +11,7 @@ class RetrieverMetadata(BaseModel): document_id: int = Field(..., description="ID of the source document") version_id: int = Field(..., description="Version ID of the source document") document_name: str = Field(..., description="Name of the source document") + url: Optional[str] = Field(..., description="URL of the source document") user_metadata: Dict[str, Any] = Field( default_factory=dict, # This will use an empty dict if None is provided description="User-defined metadata" diff --git a/eveai_chat_workers/retrievers/standard_rag.py b/eveai_chat_workers/retrievers/standard_rag.py index 61f66a6..31f6a07 100644 --- a/eveai_chat_workers/retrievers/standard_rag.py +++ b/eveai_chat_workers/retrievers/standard_rag.py @@ -97,6 +97,7 @@ class StandardRAGRetriever(BaseRetriever): query_obj = ( db.session.query( db_class, + DocumentVersion.url, (1 - db_class.embedding.cosine_distance(query_embedding)).label('similarity') ) .join(DocumentVersion, db_class.doc_vers_id == DocumentVersion.id) @@ -116,7 +117,7 @@ class StandardRAGRetriever(BaseRetriever): # Transform results into standard format processed_results = [] - for doc, similarity in results: + for doc, url, similarity in results: # Parse user_metadata to ensure it's a dictionary user_metadata = self._parse_metadata(doc.document_version.user_metadata) processed_results.append( @@ -128,6 +129,7 @@ class StandardRAGRetriever(BaseRetriever): document_id=doc.document_version.doc_id, version_id=doc.document_version.id, document_name=doc.document_version.document.name, + url=url or "", user_metadata=user_metadata, ) ) diff --git a/eveai_chat_workers/specialists/RAG_SPECIALIST/1_0.py b/eveai_chat_workers/specialists/RAG_SPECIALIST/1_0.py index 9c580cf..e72a374 100644 --- a/eveai_chat_workers/specialists/RAG_SPECIALIST/1_0.py +++ b/eveai_chat_workers/specialists/RAG_SPECIALIST/1_0.py @@ -65,7 +65,6 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor): "language": arguments.language, "query": arguments.query, "context": formatted_context, - "citations": citations, "history": self.formatted_history, "name": self.specialist.configuration.get('name', ''), "company": self.specialist.configuration.get('company', ''), diff --git a/eveai_chat_workers/specialists/SPIN_SPECIALIST/1_0.py b/eveai_chat_workers/specialists/SPIN_SPECIALIST/1_0.py index 4cff274..3b7c0cc 100644 --- a/eveai_chat_workers/specialists/SPIN_SPECIALIST/1_0.py +++ b/eveai_chat_workers/specialists/SPIN_SPECIALIST/1_0.py @@ -124,7 +124,6 @@ class SpecialistExecutor(CrewAIBaseSpecialistExecutor): "language": arguments.language, "query": arguments.query, "context": formatted_context, - "citations": citations, "history": self.formatted_history, "historic_spin": json.dumps(self.latest_spin, indent=2), "historic_lead_info": json.dumps(self.latest_lead_info, indent=2), @@ -217,7 +216,9 @@ class SPINFlow(EveAICrewAIFlow[SPINFlowState]): async def execute_rag(self): inputs = self.state.input.model_dump() try: + current_app.logger.debug("In execute_rag") crew_output = await self.rag_crew.kickoff_async(inputs=inputs) + current_app.logger.debug(f"Crew execution ended with output:\n{crew_output}") self.specialist_executor.log_tuning("RAG Crew Output", crew_output.model_dump()) output_pydantic = crew_output.pydantic if not output_pydantic: @@ -276,13 +277,16 @@ class SPINFlow(EveAICrewAIFlow[SPINFlowState]): if self.state.spin: additional_questions = additional_questions + self.state.spin.questions inputs["additional_questions"] = additional_questions + current_app.logger.debug(f"Prepared Answers: \n{inputs['prepared_answers']}") + current_app.logger.debug(f"Additional Questions: \n{additional_questions}") try: crew_output = await self.rag_consolidation_crew.kickoff_async(inputs=inputs) + current_app.logger.debug(f"Consolidation output after crew execution:\n{crew_output}") self.specialist_executor.log_tuning("RAG Consolidation Crew Output", crew_output.model_dump()) output_pydantic = crew_output.pydantic if not output_pydantic: raw_json = json.loads(crew_output.raw) - output_pydantic = LeadInfoOutput.model_validate(raw_json) + output_pydantic = RAGOutput.model_validate(raw_json) self.state.final_output = output_pydantic return crew_output except Exception as e: diff --git a/eveai_chat_workers/specialists/crewai_base_specialist.py b/eveai_chat_workers/specialists/crewai_base_specialist.py index a1bca7d..b28058d 100644 --- a/eveai_chat_workers/specialists/crewai_base_specialist.py +++ b/eveai_chat_workers/specialists/crewai_base_specialist.py @@ -314,7 +314,10 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor): ]) # Return document_ids for citations - citations = [ctx.metadata.document_id for ctx in unique_contexts] + citations = [{"document_id": ctx.metadata.document_id, + "document_version_id": ctx.metadata.version_id, + "url": ctx.metadata.url} + for ctx in unique_contexts] self.log_tuning("Context Retrieval Results", {"Formatted Context": formatted_context, @@ -345,6 +348,7 @@ class CrewAIBaseSpecialistExecutor(BaseSpecialistExecutor): modified_result = { "detailed_query": detailed_query, + "citations": citations, } final_result = result.model_copy(update=modified_result)