- Improvements working with the cloud, minio, graylog and some first bugfixing

.gitignore

@@ -3,3 +3,8 @@ docker/db/redis/
 docker/logs/
 docker/tenant_files/
 /docker/minio/
+/docker/pulling
+/docker/creating
+/docker/[internal]
+/docker/=
+/docker/stackhero/

common/utils/key_encryption.py

@@ -1,110 +0,0 @@
-from google.cloud import kms_v1
-from base64 import b64encode, b64decode
-from Crypto.Cipher import AES
-from Crypto.Random import get_random_bytes
-import random
-import time
-from flask import Flask
-import os
-import ast
-
-
-def generate_api_key(prefix="EveAI-Chat"):
-    parts = [str(random.randint(1000, 9999)) for _ in range(5)]
-    return f"{prefix}-{'-'.join(parts)}"
-
-
-class JosKMSClient(kms_v1.KeyManagementServiceClient):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.key_name = None
-        self.crypto_key = None
-        self.key_ring = None
-        self.location = None
-        self.project_id = None
-
-    def init_app(self, app: Flask):
-        self.project_id = app.config.get('GC_PROJECT_NAME')
-        self.location = app.config.get('GC_LOCATION')
-        self.key_ring = app.config.get('GC_KEY_RING')
-        self.crypto_key = app.config.get('GC_CRYPTO_KEY')
-        self.key_name = self.crypto_key_path(self.project_id, self.location, self.key_ring, self.crypto_key)
-        app.logger.info(f'Project ID: {self.project_id}')
-        app.logger.info(f'Location: {self.location}')
-        app.logger.info(f'Key Ring: {self.key_ring}')
-        app.logger.info(f'Crypto Key: {self.crypto_key}')
-        app.logger.info(f'Key Name: {self.key_name}')
-
-        app.logger.info(f'Service Account Key Path: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}')
-
-        os.environ["GOOGLE_CLOUD_PROJECT"] = self.project_id
-
-    def encrypt_api_key(self, api_key):
-        """Encrypts the API key using the latest version of the KEK."""
-        dek = get_random_bytes(32)  # AES 256-bit key
-        cipher = AES.new(dek, AES.MODE_GCM)
-        ciphertext, tag = cipher.encrypt_and_digest(api_key.encode())
-        # print(f'Dek: {dek}')
-
-        # Encrypt the DEK using the latest version of the Google Cloud KMS key
-        encrypt_response = self.encrypt(
-            request={'name': self.key_name, 'plaintext': dek}
-        )
-        encrypted_dek = encrypt_response.ciphertext
-        # print(f"Encrypted DEK: {encrypted_dek}")
-        #
-        # # Check
-        # decrypt_response = self.decrypt(
-        #     request={'name': self.key_name, 'ciphertext': encrypted_dek}
-        # )
-        # decrypted_dek = decrypt_response.plaintext
-        # print(f"Decrypted DEK: {decrypted_dek}")
-
-        # Store the version of the key used
-        key_version = encrypt_response.name
-
-        return {
-            'key_version': key_version,
-            'encrypted_dek': b64encode(encrypted_dek).decode('utf-8'),
-            'nonce': b64encode(cipher.nonce).decode('utf-8'),
-            'tag': b64encode(tag).decode('utf-8'),
-            'ciphertext': b64encode(ciphertext).decode('utf-8')
-        }
-
-    def decrypt_api_key(self, encrypted_data):
-        """Decrypts the API key using the specified key version."""
-        if isinstance(encrypted_data, str):
-            encrypted_data = ast.literal_eval(encrypted_data)
-        key_version = encrypted_data['key_version']
-        key_name = self.key_name
-        encrypted_dek = b64decode(encrypted_data['encrypted_dek'].encode('utf-8'))
-        nonce = b64decode(encrypted_data['nonce'].encode('utf-8'))
-        tag = b64decode(encrypted_data['tag'].encode('utf-8'))
-        ciphertext = b64decode(encrypted_data['ciphertext'].encode('utf-8'))
-
-        # Decrypt the DEK using the specified version of the Google Cloud KMS key
-        try:
-            decrypt_response = self.decrypt(
-                request={'name': key_name, 'ciphertext': encrypted_dek}
-            )
-            dek = decrypt_response.plaintext
-        except Exception as e:
-            print(f"Failed to decrypt DEK: {e}")
-            return None
-
-        cipher = AES.new(dek, AES.MODE_GCM, nonce=nonce)
-        api_key = cipher.decrypt_and_verify(ciphertext, tag)
-        return api_key.decode()
-
-    def check_kms_access_and_latency(self):
-        # key_name = self.crypto_key_path(self.project_id, self.location, self.key_ring, self.crypto_key)
-        #
-        # start_time = time.time()
-        # try:
-        #     response = self.get_crypto_key(name=key_name)
-        #     end_time = time.time()
-        #     print(f"Response Time: {end_time - start_time} seconds")
-        #     print("Access to KMS is successful.")
-        # except Exception as e:
-        #     print(f"Failed to access KMS: {e}")
-        pass

common/utils/simple_encryption.py

@@ -1,7 +1,13 @@
+import random
 from cryptography.fernet import Fernet
 from flask import Flask
 
 
+def generate_api_key(prefix="EveAI-Chat"):
+    parts = [str(random.randint(1000, 9999)) for _ in range(5)]
+    return f"{prefix}-{'-'.join(parts)}"
+
+
 class SimpleEncryption:
     def __init__(self, app: Flask = None):
         self.app = app

config/config.py

@@ -125,7 +125,6 @@ class DevConfig(Config):
     DEVELOPMENT = True
     DEBUG = True
     FLASK_DEBUG = True
-    PYCHARM_DEBUG = False
     EXPLAIN_TEMPLATE_LOADING = False
 
     # Database Settings
@@ -191,6 +190,7 @@ class DevConfig(Config):
     MINIO_ENDPOINT = 'minio:9000'
     MINIO_ACCESS_KEY = 'minioadmin'
     MINIO_SECRET_KEY = 'minioadmin'
+    MINIO_USE_HTTPS = False
 
 
 class ProdConfig(Config):
@@ -198,7 +198,6 @@ class ProdConfig(Config):
     DEBUG = False
     DEBUG = False
     FLASK_DEBUG = False
-    PYCHARM_DEBUG = False
     EXPLAIN_TEMPLATE_LOADING = False
 
     # Database Settings
@@ -267,6 +266,12 @@ class ProdConfig(Config):
     # PATH settings
     ffmpeg_path = '/usr/bin/ffmpeg'
 
+    # MINIO
+    MINIO_ENDPOINT = environ.get('MINIO_ENDPOINT')
+    MINIO_ACCESS_KEY = environ.get('MINIO_ACCESS_KEY')
+    MINIO_SECRET_KEY = environ.get('MINIO_SECRET_KEY')
+    MINIO_USE_HTTPS = True
+
 
 def get_config(config_name='dev'):
     configs = {

config/logging_config.py

@@ -1,3 +1,29 @@
+import os
+from graypy import GELFUDPHandler
+import logging
+import logging.config
+
+# Graylog configuration
+GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost')
+GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201))
+env = os.environ.get('FLASK_ENV', 'development')
+
+
+class CustomLogRecord(logging.LogRecord):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.component = os.environ.get('COMPONENT_NAME', 'eveai_app')  # Set default component value here
+
+
+def custom_log_record_factory(*args, **kwargs):
+    record = CustomLogRecord(*args, **kwargs)
+    return record
+
+
+# Set the custom log record factory
+logging.setLogRecordFactory(custom_log_record_factory)
+
+
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -6,7 +32,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_app.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -14,7 +40,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_workers.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -22,7 +48,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_chat.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -30,7 +56,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/eveai_chat_workers.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -38,7 +64,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/sqlalchemy.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -46,7 +72,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/mailman.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -54,7 +80,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/security.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -62,7 +88,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/rag_tuning.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -70,7 +96,7 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'logs/embed_tuning.log',
-            'maxBytes': 1024*1024*5,  # 5MB
+            'maxBytes': 1024 * 1024 * 5,  # 5MB
             'backupCount': 10,
             'formatter': 'standard',
         },
@@ -79,55 +105,69 @@ LOGGING = {
             'level': 'DEBUG',
             'formatter': 'standard',
         },
+        'graylog': {
+            'level': 'DEBUG',
+            'class': 'graypy.GELFUDPHandler',
+            'host': GRAYLOG_HOST,
+            'port': GRAYLOG_PORT,
+            'debugging_fields': True,  # Set to True if you want to include debugging fields
+            'extra_fields': True,  # Set to True if you want to include extra fields
+        },
     },
     'formatters': {
         'standard': {
-            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
+            'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
+                      '[Thread: %(threadName)s] [Host: %(hostname)s]: %(message)s'
+        },
+        'graylog': {
+            'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] '
+                      '[Thread: %(threadName)s] [Host: %(hostname)s]: %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S',
         },
     },
     'loggers': {
         'eveai_app': {  # logger for the eveai_app
-            'handlers': ['file_app',],
+            'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_workers': {  # logger for the eveai_workers
-            'handlers': ['file_workers',],
+            'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_chat': {  # logger for the eveai_chat
-            'handlers': ['file_chat',],
+            'handlers': ['file_chat', 'graylog', ] if env == 'production' else ['file_chat', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'eveai_chat_workers': {  # logger for the eveai_chat_workers
-            'handlers': ['file_chat_workers',],
+            'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'sqlalchemy.engine': {  # logger for the sqlalchemy
-            'handlers': ['file_sqlalchemy',],
+            'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'mailman': {  # logger for the mailman
-            'handlers': ['file_mailman', 'console'],
+            'handlers': ['file_mailman', 'graylog', ] if env == 'production' else ['file_mailman', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'security': {  # logger for the security
-            'handlers': ['file_security', 'console'],
+            'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'rag_tuning': {  # logger for the rag_tuning
-            'handlers': ['file_rag_tuning', 'console'],
+            'handlers': ['file_rag_tuning', 'graylog', ] if env == 'production' else ['file_rag_tuning', ],
             'level': 'DEBUG',
             'propagate': False
         },
         'embed_tuning': {  # logger for the embed_tuning
-            'handlers': ['file_embed_tuning', 'console'],
+            'handlers': ['file_embed_tuning', 'graylog', ] if env == 'production' else ['file_embed_tuning', ],
             'level': 'DEBUG',
             'propagate': False
         },
@@ -137,4 +177,4 @@ LOGGING = {
             'propagate': False
         },
     }
-}
+}

docker/build_and_push_eveai.sh

@@ -3,6 +3,8 @@
 # Exit on any error
 set -e
 
+. ./docker_env_switch.sh dev
+
 # Load environment variables
 source .env
 
@@ -15,10 +17,64 @@ TAG="latest"
 # Platforms to build for
 PLATFORMS="linux/amd64,linux/arm64"
 
-# Function to build and push a service
+# Default action
-build_and_push_service() {
+ACTION="both"
+
+# Default build options
+NO_CACHE=""
+PROGRESS=""
+DEBUG=""
+
+# Function to display usage information
+usage() {
+    echo "Usage: $0 [-b|-p] [--no-cache] [--progress=plain] [--debug] [service1 service2 ...]"
+    echo "  -b: Build only (for current platform)"
+    echo "  -p: Push only (multi-platform)"
+    echo "  --no-cache: Perform a clean build without using cache"
+    echo "  --progress=plain: Show detailed progress of the build"
+    echo "  --debug: Enable debug mode for the build"
+    echo "  If no option is provided, both build and push will be performed."
+    echo "  If no services are specified, all eveai_ services and nginx will be processed."
+}
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -b)
+            ACTION="build"
+            shift
+            ;;
+        -p)
+            ACTION="push"
+            shift
+            ;;
+        --no-cache)
+            NO_CACHE="--no-cache"
+            shift
+            ;;
+        --progress=plain)
+            PROGRESS="--progress=plain"
+            shift
+            ;;
+        --debug)
+            DEBUG="--debug"
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            usage
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Function to build and/or push a service
+process_service() {
     local SERVICE="$1"
-    echo "Building and pushing $SERVICE..."
+    echo "Processing $SERVICE..."
 
     # Extract the build context and dockerfile from the compose file
     CONTEXT=$(yq e ".services.$SERVICE.build.context" compose_dev.yaml)
@@ -36,18 +92,56 @@ build_and_push_service() {
         return 1
     fi
 
-    # Build and push
+    # Build and/or push based on ACTION
-    docker buildx build \
+    if [ "$ACTION" = "build" ]; then
-        --platform "$PLATFORMS" \
+        echo "Building $SERVICE for current platform..."
-        -t "$REGISTRY/$SERVICE:$TAG" \
+        docker build \
-        -f "$CONTEXT/$DOCKERFILE" \
+            $NO_CACHE \
-        "$CONTEXT" \
+            $PROGRESS \
-        --push
+            $DEBUG \
+            -t "$REGISTRY/$SERVICE:$TAG" \
+            -f "$CONTEXT/$DOCKERFILE" \
+            "$CONTEXT"
+    elif [ "$ACTION" = "push" ]; then
+        echo "Building and pushing $SERVICE for multiple platforms..."
+        docker buildx build \
+            $NO_CACHE \
+            $PROGRESS \
+            $DEBUG \
+            --platform "$PLATFORMS" \
+            -t "$REGISTRY/$SERVICE:$TAG" \
+            -f "$CONTEXT/$DOCKERFILE" \
+            "$CONTEXT" \
+            --push
+    else
+        echo "Building $SERVICE for current platform..."
+        docker build \
+            $NO_CACHE \
+            $PROGRESS \
+            $DEBUG \
+            -t "$REGISTRY/$SERVICE:$TAG" \
+            -f "$CONTEXT/$DOCKERFILE" \
+            "$CONTEXT"
+
+        echo "Building and pushing $SERVICE for multiple platforms..."
+        docker buildx build \
+            $NO_CACHE \
+            $PROGRESS \
+            $DEBUG \
+            --platform "$PLATFORMS" \
+            -t "$REGISTRY/$SERVICE:$TAG" \
+            -f "$CONTEXT/$DOCKERFILE" \
+            "$CONTEXT" \
+            --push
+    fi
 }
 
-# If no arguments are provided, build all services
+# If no arguments are provided, process all services
 if [ $# -eq 0 ]; then
-    mapfile -t SERVICES < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)')
+    SERVICES=()
+    while IFS= read -r line; do
+        SERVICES+=("$line")
+    done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)')
 else
     SERVICES=("$@")
 fi
@@ -65,10 +159,10 @@ docker buildx use eveai_builder
 # Loop through services
 for SERVICE in "${SERVICES[@]}"; do
     if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* ]]; then
-        if build_and_push_service "$SERVICE"; then
+        if process_service "$SERVICE"; then
-            echo "Successfully built and pushed $SERVICE"
+            echo "Successfully processed $SERVICE"
         else
-            echo "Failed to build and push $SERVICE"
+            echo "Failed to process $SERVICE"
         fi
     else
         echo "Skipping $SERVICE as it's not nginx or doesn't start with eveai_"

docker/compose_dev.yaml

@@ -29,9 +29,16 @@ x-common-variables: &common-variables
   MINIO_ENDPOINT: minio:9000
   MINIO_ACCESS_KEY: minioadmin
   MINIO_SECRET_KEY: minioadmin
+  NGINX_SERVER_NAME: 'localhost http://macstudio.ask-eve-ai-local.com/'
+
+
+networks:
+  eveai-network:
+    driver: bridge
 
 services:
   nginx:
+    image: josakola/nginx:latest
     build:
       context: ..
       dockerfile: ./docker/nginx/Dockerfile
@@ -41,6 +48,8 @@ services:
     ports:
       - 80:80
       - 8080:8080
+    environment:
+      <<: *common-variables
     volumes:
       - ../nginx:/etc/nginx
       - ../nginx/sites-enabled:/etc/nginx/sites-enabled
@@ -50,8 +59,11 @@ services:
     depends_on:
       - eveai_app
       - eveai_chat
+    networks:
+      - eveai-network
 
   eveai_app:
+    image: josakola/eveai_app:latest
     build:
       context: ..
       dockerfile: ./docker/eveai_app/Dockerfile
@@ -62,6 +74,7 @@ services:
       - 5001:5001
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_app
     volumes:
       - ../eveai_app:/app/eveai_app
       - ../common:/app/common
@@ -69,7 +82,7 @@ services:
       - ../migrations:/app/migrations
       - ../scripts:/app/scripts
       - ../patched_packages:/app/patched_packages
-      - ./logs:/app/logs
+      - eveai_logs:/app/logs
     depends_on:
        db:
          condition: service_healthy
@@ -82,9 +95,13 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
-    command: ["sh", "-c", "scripts/start_eveai_app.sh"]
+#    entrypoint: ["scripts/entrypoint.sh"]
+#    command: ["scripts/start_eveai_app.sh"]
+    networks:
+      - eveai-network
 
   eveai_workers:
+    image: josakola/eveai_workers:latest
     build:
       context: ..
       dockerfile: ./docker/eveai_workers/Dockerfile
@@ -95,13 +112,14 @@ services:
 #      - 5001:5001
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_workers
     volumes:
       - ../eveai_workers:/app/eveai_workers
       - ../common:/app/common
       - ../config:/app/config
       - ../scripts:/app/scripts
       - ../patched_packages:/app/patched_packages
-      - ./logs:/app/logs
+      - eveai_logs:/app/logs
     depends_on:
       db:
         condition: service_healthy
@@ -114,9 +132,13 @@ services:
 #      interval: 10s
 #      timeout: 5s
 #      retries: 5
-    command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
+#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
+#    command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
+    networks:
+      - eveai-network
 
   eveai_chat:
+    image: josakola/eveai_chat:latest
     build:
       context: ..
       dockerfile: ./docker/eveai_chat/Dockerfile
@@ -127,13 +149,14 @@ services:
       - 5002:5002
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_chat
     volumes:
       - ../eveai_chat:/app/eveai_chat
       - ../common:/app/common
       - ../config:/app/config
       - ../scripts:/app/scripts
       - ../patched_packages:/app/patched_packages
-      - ./logs:/app/logs
+      - eveai_logs:/app/logs
     depends_on:
       db:
         condition: service_healthy
@@ -144,9 +167,13 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
-    command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
+#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
+#    command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
+    networks:
+      - eveai-network
 
   eveai_chat_workers:
+    image: josakola/eveai_chat_workers:latest
     build:
       context: ..
       dockerfile: ./docker/eveai_chat_workers/Dockerfile
@@ -157,13 +184,14 @@ services:
 #      - 5001:5001
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_chat_workers
     volumes:
       - ../eveai_chat_workers:/app/eveai_chat_workers
       - ../common:/app/common
       - ../config:/app/config
       - ../scripts:/app/scripts
       - ../patched_packages:/app/patched_packages
-      - ./logs:/app/logs
+      - eveai_logs:/app/logs
     depends_on:
       db:
         condition: service_healthy
@@ -174,7 +202,10 @@ services:
 #      interval: 10s
 #      timeout: 5s
 #      retries: 5
-    command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
+#    entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
+#    command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
+    networks:
+      - eveai-network
 
   db:
     hostname: db
@@ -194,6 +225,8 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
+    networks:
+      - eveai-network
 
   redis:
     image: redis:7.2.5
@@ -207,6 +240,8 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
+    networks:
+      - eveai-network
 
   minio:
     image: minio/minio
@@ -228,9 +263,12 @@ services:
       timeout: 20s
       retries: 3
       start_period: 30s
+    networks:
+      - eveai-network
 
 volumes:
   minio_data:
+  eveai_logs:
 #  db-data:
 #  redis-data:
 #  tenant-files:

docker/compose_stackhero.yaml

@@ -20,16 +20,26 @@ x-common-variables: &common-variables
   SECURITY_PASSWORD_SALT: '166448071751628781809462050022558634074'
   MAIL_USERNAME: 'evie_admin@askeveai.com'
   MAIL_PASSWORD: 's5D%R#y^v!s&6Z^i0k&'
-  REDIS_USER: admin
+  REDIS_USER: eveai
-  REDIS_PASS: 'b32vtDtLriSY1fL2zGrZg8IZKI0g9ucsLtVNanRFAras6oZ51wjVNB1Y05uG7uEw'
+  REDIS_PASS: 'jHliZwGD36sONgbm0fc6SOpzLbknqq4RNF8K'
   REDIS_URL: 8bciqc.stackhero-network.com
   REDIS_PORT: '9961'
   OPENAI_API_KEY: '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'
   GROQ_API_KEY: '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'
   ANTHROPIC_API_KEY: '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'
-  PORTKEY_API_KEY: '3C+zAGR8pCalevBXFVc0l8R2MPYc'
+  PORTKEY_API_KEY: 'XvmvBFIVbm76opUxA7MNP14QmdQj'
   JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408'
   API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0='
+  GRAYLOG_HOST: de4zvu.stackhero-network.com
+  GRAYLOG_PORT: '12201'
+  MINIO_ENDPOINT: 'fxwnyl.stackhero-network.com:443'
+  MINIO_ACCESS_KEY: 04JKmQln8PQpyTmMiCPc
+  MINIO_SECRET_KEY: 2PEZAD1nlpAmOyDV0TUTuJTQw1qVuYLF3A7GMs0D
+  NGINX_SERVER_NAME: 'evie.askeveai.com mxz536.stackhero-network.com'
+
+networks:
+  eveai-network:
+    driver: bridge
 
 services:
   nginx:
@@ -38,19 +48,25 @@ services:
     ports:
       - 80:80
       - 8080:8080
+    environment:
+      <<: *common-variables
     volumes:
 #      - ../nginx:/etc/nginx
 #      - ../nginx/sites-enabled:/etc/nginx/sites-enabled
 #      - ../nginx/static:/etc/nginx/static
 #      - ../nginx/public:/etc/nginx/public
-      - logs:/var/log/nginx
+      - eveai_logs:/var/log/nginx
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.api.rule=Host(`evie.askeveai.com`)"
+      - "traefik.http.routers.nginx.rule=Host(`evie.askeveai.com`) || Host(`mxz536.stackhero-network.com`)"
+      - "traefik.http.routers.nginx.entrypoints=web,websecure"
+      - "traefik.http.routers.nginx.tls.certresolver=myresolver"
       - "traefik.http.services.nginx.loadbalancer.server.port=80"
     depends_on:
       - eveai_app
       - eveai_chat
+    networks:
+      - eveai-network
 
   eveai_app:
     platform: linux/amd64
@@ -59,14 +75,16 @@ services:
       - 5001:5001
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_app
     volumes:
-      - logs:/app/logs
+      - eveai_logs:/app/logs
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
       interval: 10s
       timeout: 5s
       retries: 5
-    command: ["sh", "-c", "scripts/start_eveai_app.sh"]
+    networks:
+      - eveai-network
 
   eveai_workers:
     platform: linux/amd64
@@ -75,14 +93,16 @@ services:
 #      - 5001:5001
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_workers
     volumes:
-      - logs:/app/logs
+      - eveai_logs:/app/logs
 #    healthcheck:
 #      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
 #      interval: 10s
 #      timeout: 5s
 #      retries: 5
-    command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
+    networks:
+      - eveai-network
 
   eveai_chat:
     platform: linux/amd64
@@ -91,14 +111,16 @@ services:
       - 5002:5002
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_chat
     volumes:
-      - logs:/app/logs
+      - eveai_logs:/app/logs
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ]  # Adjust based on your health endpoint
       interval: 10s
       timeout: 5s
       retries: 5
-    command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
+    networks:
+      - eveai-network
 
   eveai_chat_workers:
     platform: linux/amd64
@@ -107,18 +129,19 @@ services:
 #      - 5001:5001
     environment:
       <<: *common-variables
+      COMPONENT_NAME: eveai_chat_workers
     volumes:
-      - logs:/app/logs
+      - eveai_logs:/app/logs
 #    healthcheck:
 #      test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
 #      interval: 10s
 #      timeout: 5s
 #      retries: 5
-    command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
+    networks:
-
+      - eveai-network
 
 volumes:
-  logs:
+  eveai_logs:
 #  miniAre theo_data:
 #  db-data:
 #  redis-data:

docker/copy_docker_logs.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Check if a volume name was provided
+if [ $# -eq 0 ]; then
+    echo "Error: No volume name provided."
+    echo "Usage: $0 <volume_name>"
+    exit 1
+fi
+
+# Get the volume name from the first argument
+VOLUME_NAME=$1
+
+# Define the local directory to copy logs to
+LOCAL_DIR="./stackhero/logs"
+
+# Create the local directory if it doesn't exist
+mkdir -p "$LOCAL_DIR"
+
+echo "Copying logs from Docker volume '$VOLUME_NAME' to $LOCAL_DIR"
+
+# Create a temporary container with the volume mounted
+CONTAINER_ID=$(docker run -d -v $VOLUME_NAME:/logs --name temp_log_container alpine sh -c 'tail -f /dev/null')
+
+# Check if the container was created successfully
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to create temporary container."
+    exit 1
+fi
+
+# Copy files from the container to the local machine
+docker cp $CONTAINER_ID:/logs/. "$LOCAL_DIR"
+
+# Check if the copy was successful
+if [ $? -eq 0 ]; then
+    echo "Logs successfully copied to $LOCAL_DIR"
+else
+    echo "Error: Failed to copy logs from the container."
+fi
+
+# Remove the temporary container
+docker rm -f $CONTAINER_ID
+
+echo "Temporary container removed."

docker/docker_env_switch.sh

@@ -48,10 +48,12 @@ echo "Set COMPOSE_FILE to $COMPOSE_FILE"
 # Define aliases for common Docker commands
 alias docker-compose="docker compose -f $COMPOSE_FILE"
 alias dc="docker compose -f $COMPOSE_FILE"
-alias dcup="docker compose -f $COMPOSE_FILE up -d"
+alias dcup="docker compose -f $COMPOSE_FILE up -d --no-build --remove-orphans"
 alias dcdown="docker compose -f $COMPOSE_FILE down"
 alias dcps="docker compose -f $COMPOSE_FILE ps"
 alias dclogs="docker compose -f $COMPOSE_FILE logs"
+alias dcpull="docker compose -f $COMPOSE_FILE pull"
+alias dcrefresh="docker compose -f $COMPOSE_FILE pull"
 
 echo "Docker environment switched to $1"
-echo "You can now use 'docker-compose', 'dc', 'dcup', 'dcdown', 'dcps', and 'dclogs' commands"
+echo "You can now use 'docker-compose', 'dc', 'dcup', 'dcdown', 'dcps', 'dclogs' , dcpull or dcrefresh commands"

docker/eveai_app/Dockerfile

@@ -24,7 +24,7 @@ RUN adduser \
     --disabled-password \
     --gecos "" \
     --home "/nonexistent" \
-    --shell "/sbin/nologin" \
+    --shell "/bin/bash" \
     --no-create-home \
     --uid "${UID}" \
     appuser
@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Create logs directory and set permissions
+RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+
 # Download dependencies as a separate step to take advantage of Docker's caching.
 # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
 # Leverage a bind mount to requirements.txt to avoid having to copy them into
@@ -45,9 +48,6 @@ RUN apt-get update && apt-get install -y \
 COPY requirements.txt /app/
 RUN python -m pip install -r /app/requirements.txt
 
-# Switch to the non-privileged user to run the application.
-USER appuser
-
 # Copy the source code into the container.
 COPY eveai_app /app/eveai_app
 COPY common /app/common
@@ -56,8 +56,15 @@ COPY migrations /app/migrations
 COPY scripts /app/scripts
 COPY patched_packages /app/patched_packages
 
+# Set permissions for entrypoint script
+RUN chmod 777 /app/scripts/entrypoint.sh
+
+# Set ownership of the application directory to the non-privileged user
+RUN chown -R appuser:appuser /app
+
 # Expose the port that the application listens on.
 EXPOSE 5001
 
-# Run the application.
+# Set entrypoint and command
-#CMD ["sh", "-c", "/app/scripts/start_eveai_app.sh"]
+ENTRYPOINT ["/app/scripts/entrypoint.sh"]
+CMD ["/app/scripts/start_eveai_app.sh"]

docker/eveai_chat/Dockerfile

@@ -24,7 +24,7 @@ RUN adduser \
     --disabled-password \
     --gecos "" \
     --home "/nonexistent" \
-    --shell "/sbin/nologin" \
+    --shell "/bin/bash" \
     --no-create-home \
     --uid "${UID}" \
     appuser
@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Create logs directory and set permissions
+RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+
 # Download dependencies as a separate step to take advantage of Docker's caching.
 # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
 # Leverage a bind mount to requirements.txt to avoid having to copy them into
@@ -45,9 +48,6 @@ RUN apt-get update && apt-get install -y \
 COPY ../../requirements.txt /app/
 RUN python -m pip install -r requirements.txt
 
-# Switch to the non-privileged user to run the application.
-USER appuser
-
 # Copy the source code into the container.
 COPY eveai_chat /app/eveai_chat
 COPY common /app/common
@@ -55,8 +55,15 @@ COPY config /app/config
 COPY scripts /app/scripts
 COPY patched_packages /app/patched_packages
 
+# Set permissions for entrypoint script
+RUN chmod 777 /app/scripts/entrypoint.sh
+
+# Set ownership of the application directory to the non-privileged user
+RUN chown -R appuser:appuser /app
+
 # Expose the port that the application listens on.
 EXPOSE 5002
 
-# Run the application.
+# Set entrypoint and command
-#CMD ["sh", "-c", "/app/scripts/start_eveai_chat.sh"]
+ENTRYPOINT ["/app/scripts/entrypoint.sh"]
+CMD ["/app/scripts/start_eveai_chat.sh"]

docker/eveai_chat_workers/Dockerfile

@@ -24,7 +24,7 @@ RUN adduser \
     --disabled-password \
     --gecos "" \
     --home "/nonexistent" \
-    --shell "/sbin/nologin" \
+    --shell "/bin/bash" \
     --no-create-home \
     --uid "${UID}" \
     appuser
@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Create logs directory and set permissions
+RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+
 # Download dependencies as a separate step to take advantage of Docker's caching.
 # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
 # Leverage a bind mount to requirements.txt to avoid having to copy them into
@@ -45,18 +48,20 @@ RUN apt-get update && apt-get install -y \
 COPY requirements.txt /app/
 RUN python -m pip install -r /app/requirements.txt
 
-# Switch to the non-privileged user to run the application.
-USER appuser
-
 # Copy the source code into the container.
 COPY eveai_chat_workers /app/eveai_chat_workers
 COPY common /app/common
 COPY config /app/config
 COPY scripts /app/scripts
 COPY patched_packages /app/patched_packages
+COPY --chown=root:root scripts/entrypoint.sh /app/scripts/
 
-# Expose the port that the application listens on.
+# Set permissions for entrypoint script
-#EXPOSE 5001
+RUN chmod 777 /app/scripts/entrypoint.sh
 
-# Run the application.
+# Set ownership of the application directory to the non-privileged user
-# CMD ["sh", "-c", "/app/scripts/start_eveai_workers.sh"]
+RUN chown -R appuser:appuser /app
+
+# Set entrypoint and command
+ENTRYPOINT ["/app/scripts/entrypoint.sh"]
+CMD ["/app/scripts/start_eveai_chat_workers.sh"]

docker/eveai_workers/Dockerfile

@@ -24,7 +24,7 @@ RUN adduser \
     --disabled-password \
     --gecos "" \
     --home "/nonexistent" \
-    --shell "/sbin/nologin" \
+    --shell "/bin/bash" \
     --no-create-home \
     --uid "${UID}" \
     appuser
@@ -38,6 +38,9 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Create logs directory and set permissions
+RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
+
 # Install Python dependencies.
 
 # Download dependencies as a separate step to take advantage of Docker's caching.
@@ -48,18 +51,20 @@ RUN apt-get update && apt-get install -y \
 COPY requirements.txt /app/
 RUN python -m pip install -r /app/requirements.txt
 
-# Switch to the non-privileged user to run the application.
-USER appuser
-
 # Copy the source code into the container.
 COPY eveai_workers /app/eveai_workers
 COPY common /app/common
 COPY config /app/config
 COPY scripts /app/scripts
 COPY patched_packages /app/patched_packages
+COPY --chown=root:root scripts/entrypoint.sh /app/scripts/
 
-# Expose the port that the application listens on.
+# Set permissions for entrypoint script
-#EXPOSE 5001
+RUN chmod 777 /app/scripts/entrypoint.sh
 
-# Run the application.
+# Set ownership of the application directory to the non-privileged user
-# CMD ["sh", "-c", "/app/scripts/start_eveai_workers.sh"]
+RUN chown -R appuser:appuser /app
+
+# Set entrypoint and command
+ENTRYPOINT ["/app/scripts/entrypoint.sh"]
+CMD ["/app/scripts/start_eveai_workers.sh"]

eveai_app/views/user_views.py

@@ -13,7 +13,7 @@ from common.utils.security_utils import send_confirmation_email, send_reset_emai
 from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm
 from common.utils.database import Database
 from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed
-from common.utils.key_encryption import generate_api_key
+from common.utils.simple_encryption import generate_api_key
 from common.utils.nginx_utils import prefixed_url_for
 
 user_bp = Blueprint('user_bp', __name__, url_prefix='/user')

eveai_workers/tasks.py

@@ -275,14 +275,16 @@ def summarize_chunk(tenant, model_variables, document_version, chunk):
     llm = model_variables['llm']
     template = model_variables['summary_template']
     language_template = create_language_template(template, document_version.language)
-    current_app.logger.debug(f'Language prompt: {language_template}')
+    summary_prompt = ChatPromptTemplate.from_template(language_template)
-    chain = load_summarize_chain(llm, chain_type='stuff', prompt=ChatPromptTemplate.from_template(language_template))
+    setup = RunnablePassthrough()
+    output_parser = StrOutputParser()
 
-    doc_creator = CharacterTextSplitter(chunk_size=model_variables['max_chunk_size'] * 2, chunk_overlap=0)
+    chain = setup | summary_prompt | llm | output_parser
-    text_to_summarize = doc_creator.create_documents(chunk)
 
     try:
-        summary = chain.invoke({"text": text_to_summarize})
+        current_app.logger.debug(f'Starting summarizing chunk for tenant {tenant.id} '
+                                 f'on document version {document_version.id}')
+        summary = chain.invoke({"text": chunk})
         current_app.logger.debug(f'Finished summarizing chunk for tenant {tenant.id} '
                                  f'on document version {document_version.id}.')
         return summary

nginx/nginx.conf

@@ -42,7 +42,7 @@ http {
     server {
         listen       80;
         listen       8080;
-        server_name  localhost macstudio.ask-eve-ai-local.com;
+        server_name  ${NGINX_SERVER_NAME};
 
         #charset koi8-r;
 

requirements.txt

@@ -22,12 +22,6 @@ Flask-SQLAlchemy~=3.1.1
 Flask-WTF~=1.2.1
 gevent~=24.2.1
 gevent-websocket~=0.10.1
-google~=3.0.0
-google-api-core~=2.19.1rc0
-google-auth~=2.30.0
-google-cloud-core~=2.4.1
-google-cloud-kms~=2.23.0
-googleapis-common-protos~=1.63.2rc0
 greenlet~=3.0.3
 gunicorn~=22.0.0
 Jinja2~=3.1.4
@@ -48,7 +42,6 @@ pg8000~=1.31.2
 pgvector~=0.2.5
 pycryptodome~=3.20.0
 pydantic~=2.7.4
-pydevd-pycharm~=242.18071.12
 PyJWT~=2.8.0
 pypdf~=4.2.0
 PySocks~=1.7.1
@@ -79,4 +72,5 @@ portkey_ai~=1.7.0
 minio~=7.2.7
 Werkzeug~=3.0.3
 itsdangerous~=2.2.0
-cryptography~=43.0.0
+cryptography~=43.0.0
+graypy~=2.1.0

scripts/entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+# Ensure the logs directory has the correct permissions
+echo "Changing permissions on logs directory"
+#chown -R appuser:appuser /app/logs
+chmod -R 777 /app/logs
+
+# Wait for the database to be ready
+until pg_isready -h $DB_HOST -p $DB_PORT; do
+  echo "Postgres is unavailable - sleeping"
+  sleep 2
+done
+
+# Switch to appuser and execute the command passed to the script
+exec su appuser -c "$@"

scripts/start_eveai_app.sh

@@ -3,20 +3,23 @@
 cd "/app" || exit 1
 export PYTHONPATH="$PYTHONPATH:/app/"
 
+# Ensure we can write the logs
+chown -R appuser:appuser /app/logs
+
 # Wait for the database to be ready
 echo "Waiting for database to be ready"
-until pg_isready -h db -p 5432; do
+until pg_isready -h $DB_HOST -p $DB_PORT; do
   echo "Postgres is unavailable - sleeping"
   sleep 2
 done
 echo "Postgres is up - executing commands"
 
-export PGPASSWORD=Skywalker!
+export PGPASSWORD=$DB_PASS
 # Check if the database exists and initialize if not
-if ! psql -U luke -h db -d eveai -c '\dt' | grep -q 'No relations found'; then
+if ! psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d $DB_NAME -c '\dt' | grep -q 'No relations found'; then
   echo "Database eveai does not exist or is empty. Initializing..."
-  psql -U luke -h db -d postgres -c "CREATE DATABASE eveai;"
+  psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -c "CREATE DATABASE $DB_NAME;"
-  psql -U luke -h db -d eveai -c "CREATE EXTENSION IF NOT EXISTS vector;"
+  psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d $DB_NAME -c "CREATE EXTENSION IF NOT EXISTS vector;"
 fi
 
 echo "Applying migrations to the public and tenant schema..."

scripts/start_eveai_chat.sh

@@ -4,6 +4,9 @@ cd "/app/" || exit 1
 export PROJECT_DIR="/app"
 export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR"  # Include the app directory in the Python path & patched packages
 
+# Ensure we can write the logs
+chown -R appuser:appuser /app/logs
+
 # Set flask environment variables
 #export FLASK_ENV=development  # Use 'production' as appropriate
 #export FLASK_DEBUG=1  # Use 0 for production

scripts/start_eveai_chat_workers.sh

@@ -4,6 +4,9 @@ cd "/app/" || exit 1
 export PROJECT_DIR="/app"
 export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR"  # Include the app directory in the Python path & patched packages
 
+# Ensure we can write the logs
+chown -R appuser:appuser /app/logs
+
 # Start a worker for the 'llm_interactions' queue with auto-scaling
 celery -A eveai_chat_workers.celery worker --loglevel=info -Q llm_interactions --autoscale=2,8 --hostname=interactions_worker@%h &
 

scripts/start_eveai_workers.sh

@@ -4,6 +4,9 @@ cd "/app/" || exit 1
 export PROJECT_DIR="/app"
 export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR"  # Include the app directory in the Python path & patched packages
 
+# Ensure we can write the logs
+chown -R appuser:appuser /app/logs
+
 # Start a worker for the 'embeddings' queue with higher concurrency
 celery -A eveai_workers.celery worker --loglevel=info -Q embeddings --autoscale=2,8 --hostname=embeddings_worker@%h &