- Add test environment to __init__.py for all eveai services

- Add postgresql certificate to secrets for secure communication in staging and production environments
- Adapt for TLS communication with PostgreSQL
- Adapt tasks to handle invalid connections from the connection pool
- Migrate to psycopg3 for connection to PostgreSQL

eveai_workers/__init__.py

@@ -17,6 +17,10 @@ def create_app(config_file=None):
     match environment:
         case 'development':
             app.config.from_object(get_config('dev'))
+        case 'test':
+            app.config.from_object(get_config('test'))
+        case 'staging':
+            app.config.from_object(get_config('staging'))
         case 'production':
             app.config.from_object(get_config('prod'))
         case _:

eveai_workers/tasks.py

@@ -10,7 +10,7 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.runnables import RunnablePassthrough
 from sqlalchemy import or_
-from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.exc import SQLAlchemyError, InterfaceError, OperationalError
 import traceback
 
 from common.extensions import db, cache_manager
@@ -37,8 +37,10 @@ def ping():
     return 'pong'
 
 
-@current_celery.task(name='create_embeddings', queue='embeddings')
-def create_embeddings(tenant_id, document_version_id):
+@current_celery.task(bind=True, name='create_embeddings', queue='embeddings',
+                      autoretry_for=(InterfaceError, OperationalError),
+                      retry_backoff=True, retry_jitter=True, max_retries=5)
+def create_embeddings(self, tenant_id, document_version_id):
     document_version = None
     try:
         # Retrieve Tenant for which we are processing
@@ -127,7 +129,7 @@ def create_embeddings(tenant_id, document_version_id):
             document_version.processing_finished_at = dt.now(tz.utc)
             document_version.processing_error = str(e)[:255]
             db.session.commit()
-            create_embeddings.update_state(state=states.FAILURE)
+            self.update_state(state=states.FAILURE)
             raise