Refactoring part 1

Some changes for workers, but stopped due to refactoring

config/config.py

@@ -0,0 +1,107 @@
+from os import environ, path
+from datetime import timedelta
+
+basedir = path.abspath(path.dirname(__file__))
+
+
+class Config(object):
+    DEBUG = False
+    DEVELOPMENT = False
+    SECRET_KEY = '97867c1491bea5ee6a8e8436eb11bf2ba6a69ff53ab1b17ecba450d0f2e572e1'
+    SESSION_COOKIE_SECURE = True
+    SESSION_COOKIE_HTTPONLY = True
+
+    # WTF_CSRF_ENABLED = True
+
+    # flask-security-too settings
+    SECURITY_PASSWORD_SALT = '228614859439123264035565568761433607235'
+    REMEMBER_COOKIE_SAMESITE = 'strict'
+    SESSION_COOKIE_SAMESITE = 'strict'
+    SECURITY_CONFIRMABLE = True
+    SECURITY_TRACKABLE = True
+    SECURITY_PASSWORD_COMPLEXITY_CHECKER = 'zxcvbn'
+    SECURITY_POST_LOGIN_VIEW = '/user/tenant'
+    SECURITY_RECOVERABLE = True
+    SECURITY_EMAIL_SENDER = "eveai_super@flow-it.net"
+    PERMANENT_SESSION_LIFETIME = timedelta(minutes=60)
+    SESSION_REFRESH_EACH_REQUEST = True
+
+    # flask-mailman settings
+    MAIL_SERVER = 'mail.flow-it.net'
+    MAIL_PORT = 587
+    MAIL_USE_TLS = True
+    MAIL_USE_SSL = False
+    MAIL_DEFAULT_SENDER = ('eveAI Admin', 'eveai_admin@flow-it.net')
+
+    # file upload settings
+    MAX_CONTENT_LENGTH = 16 * 1024 * 1024
+    UPLOAD_EXTENSIONS = ['.txt', '.pdf', '.png', '.jpg', '.jpeg', '.gif']
+
+    # supported languages
+    SUPPORTED_LANGUAGES = ['en', 'fr', 'nl', 'de', 'es']
+
+    # supported LLMs
+    SUPPORTED_EMBEDDINGS = ['openai.text-embedding-3-small', 'mistral.mistral-embed']
+    SUPPORTED_LLMS = ['openai.gpt-4-turbo', 'openai.gpt-3.5-turbo', 'mistral.mistral-large-2402']
+
+    # Celery settings
+    CELERY_TASK_SERIALIZER = 'json'
+    CELERY_ACCEPT_CONTENT = ['json']
+    CELERY_TIMEZONE = 'UTC'
+    CELERY_ENABLE_UTC = True
+
+    # Define multiple queues
+    CELERY_TASK_QUEUES = {
+        'embeddings': {
+            'exchange': 'embeddings',
+            'routing_key': 'embeddings.key',
+            'queue_arguments': {'x-max-priority': 10}
+        },
+        'llm_interactions': {
+            'exchange': 'llm_interactions',
+            'routing_key': 'llm_interactions.key',
+            'queue_arguments': {'x-max-priority': 5}
+        }
+    }
+
+
+class DevConfig(Config):
+    DEVELOPMENT = True
+    DEBUG = True
+    FLASK_DEBUG = True
+    SQLALCHEMY_DATABASE_URI = 'postgresql+pg8000://josako@localhost:5432/eveAI'
+    SQLALCHEMY_BINDS = {'public': 'postgresql+pg8000://josako@localhost:5432/eveAI'}
+    EXPLAIN_TEMPLATE_LOADING = False
+
+    # flask-mailman settings
+    MAIL_USERNAME = 'eveai_super@flow-it.net'
+    MAIL_PASSWORD = '$6xsWGbNtx$CFMQZqc*'
+
+    # file upload settings
+    UPLOAD_FOLDER = '/Volumes/OWC4M2_1/Development/eveAI/file_store'
+
+    # Celery settings
+    CELERY_BROKER_URL = 'redis://localhost:6379/0'  # Default Redis configuration
+    CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
+
+    # OpenAI API Keys
+    OPENAI_API_KEY = 'sk-proj-8R0jWzwjL7PeoPyMhJTZT3BlbkFJLb6HfRB2Hr9cEVFWEhU7'
+
+    # Unstructured settings
+    UNSTRUCTURED_API_KEY = 'pDgCrXumYhM3CNvjvwV8msMldXC3uw'
+    UNSTRUCTURED_BASE_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io'
+    UNSTRUCTURED_FULL_URL = 'https://flowitbv-16c4us0m.api.unstructuredapp.io/general/v0/general'
+
+
+class ProdConfig(Config):
+    DEVELOPMENT = False
+    DEBUG = False
+    # SQLALCHEMY_DATABASE_URI = environ.get('SQLALCHEMY_DATABASE_URI') or \
+    #                           'sqlite:///' + os.path.join(basedir, 'db.sqlite')
+
+
+config = {
+    'dev': DevConfig(),
+    'prod': ProdConfig(),
+    'default': DevConfig(),
+}

config/logging_config.py

@@ -0,0 +1,31 @@
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'file': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'app.log',
+            'maxBytes': 1024*1024*5,  # 5MB
+            'backupCount': 10,
+            'formatter': 'standard',
+        },
+        'console': {
+            'class': 'logging.StreamHandler',
+            'level': 'DEBUG',
+            'formatter': 'standard',
+        },
+    },
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
+        },
+    },
+    'loggers': {
+        '': {  # root logger
+            'handlers': ['file', 'console'],
+            'level': 'DEBUG',
+            'propagate': True
+        }
+    }
+}