pieter/eveAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring finished :-)

Browse Source 
eveai_workers now working (with errors ;-) )
Remote debugging now available
This commit is contained in:
Josako
2024-05-07 22:51:48 +02:00
parent 494d5508ae
commit cd5afa0408
14 changed files with 147 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
eveai_workers/tasks.py
View File

@@ -1,16 +1,18 @@
from datetime import datetime as dt, timezone as tz
from flask import current_app
from langchain_community.document_loaders.unstructured import UnstructuredAPIFileLoader
from flask import current_app
import os
from celery import shared_task
from common.utils.database import Database
from common.models.document import DocumentVersion, EmbeddingMistral, EmbeddingSmallOpenAI
from eveai_app import db
from common.extensions import db
from common.utils.celery_utils import current_celery
@shared_task(name='create_embeddings', queue='embeddings')
@current_celery.task(name='create_embeddings', queue='embeddings')
def create_embeddings(tenant_id, document_version_id, default_embedding_model):
import pydevd_pycharm
pydevd_pycharm.settrace('localhost', port=50170, stdoutToServer=True, stderrToServer=True)
current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id} '
f'with model {default_embedding_model}')
@@ -45,7 +47,7 @@ def create_embeddings(tenant_id, document_version_id, default_embedding_model):
api_key = current_app.config.get('UNSTRUCTURED_API_KEY')
file_path = os.path.join(current_app.config['UPLOAD_FOLDER'],
document_version.file_location,
document_version.file_path)
document_version.file_name)
with open(file_path, 'rb') as f:
loader = UnstructuredAPIFileLoader(f,
url=url,
@@ -61,7 +63,7 @@ def create_embeddings(tenant_id, document_version_id, default_embedding_model):
print(documents)
@shared_task(name='ask_eve_ai', queue='llm_interactions')
@current_celery.task(name='ask_eve_ai', queue='llm_interactions')
def ask_eve_ai(query):
# Interaction logic with LLMs like GPT (Langchain API calls, etc.)
pass