Refactoring finished :-)

eveai_workers now working (with errors ;-) )
Remote debugging now available

eveai_workers/__init__.py

@@ -0,0 +1,33 @@
+import logging
+import logging.config
+from flask import Flask
+
+from common.utils.celery_utils import make_celery, init_celery
+from common.extensions import db
+from config.logging_config import LOGGING
+
+
+def create_app(config_file=None):
+    app = Flask(__name__)
+
+    if config_file is None:
+        app.config.from_object('config.config.DevConfig')
+    else:
+        app.config.from_object(config_file)
+
+    logging.config.dictConfig(LOGGING)
+    register_extensions(app)
+
+    celery = make_celery(app.name, app.config)
+    init_celery(celery, app)
+
+    from . import tasks
+
+    return app, celery
+
+
+def register_extensions(app):
+    db.init_app(app)
+
+
+app, celery = create_app()

eveai_workers/tasks.py

@@ -1,16 +1,18 @@
 from datetime import datetime as dt, timezone as tz
-from flask import current_app
 from langchain_community.document_loaders.unstructured import UnstructuredAPIFileLoader
+from flask import current_app
 import os
-from celery import shared_task
 
 from common.utils.database import Database
 from common.models.document import DocumentVersion, EmbeddingMistral, EmbeddingSmallOpenAI
-from eveai_app import db
+from common.extensions import db
+from common.utils.celery_utils import current_celery
 
 
-@shared_task(name='create_embeddings', queue='embeddings')
+@current_celery.task(name='create_embeddings', queue='embeddings')
 def create_embeddings(tenant_id, document_version_id, default_embedding_model):
+    import pydevd_pycharm
+    pydevd_pycharm.settrace('localhost', port=50170, stdoutToServer=True, stderrToServer=True)
     current_app.logger.info(f'Creating embeddings for tenant {tenant_id} on document version {document_version_id} '
                             f'with model {default_embedding_model}')
 
@@ -45,7 +47,7 @@ def create_embeddings(tenant_id, document_version_id, default_embedding_model):
             api_key = current_app.config.get('UNSTRUCTURED_API_KEY')
             file_path = os.path.join(current_app.config['UPLOAD_FOLDER'],
                                      document_version.file_location,
-                                     document_version.file_path)
+                                     document_version.file_name)
             with open(file_path, 'rb') as f:
                 loader = UnstructuredAPIFileLoader(f,
                                                    url=url,
@@ -61,7 +63,7 @@ def create_embeddings(tenant_id, document_version_id, default_embedding_model):
                 print(documents)
 
 
-@shared_task(name='ask_eve_ai', queue='llm_interactions')
+@current_celery.task(name='ask_eve_ai', queue='llm_interactions')
 def ask_eve_ai(query):
     # Interaction logic with LLMs like GPT (Langchain API calls, etc.)
     pass