diff --git a/.gitignore b/.gitignore index 0db0019..fed4a02 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ docker/db/redis/ docker/logs/ docker/tenant_files/ /docker/minio/ +/docker/pulling +/docker/creating +/docker/[internal] +/docker/= +/docker/stackhero/ diff --git a/common/utils/key_encryption.py b/common/utils/key_encryption.py deleted file mode 100644 index 5ee1bec..0000000 --- a/common/utils/key_encryption.py +++ /dev/null @@ -1,110 +0,0 @@ -from google.cloud import kms_v1 -from base64 import b64encode, b64decode -from Crypto.Cipher import AES -from Crypto.Random import get_random_bytes -import random -import time -from flask import Flask -import os -import ast - - -def generate_api_key(prefix="EveAI-Chat"): - parts = [str(random.randint(1000, 9999)) for _ in range(5)] - return f"{prefix}-{'-'.join(parts)}" - - -class JosKMSClient(kms_v1.KeyManagementServiceClient): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.key_name = None - self.crypto_key = None - self.key_ring = None - self.location = None - self.project_id = None - - def init_app(self, app: Flask): - self.project_id = app.config.get('GC_PROJECT_NAME') - self.location = app.config.get('GC_LOCATION') - self.key_ring = app.config.get('GC_KEY_RING') - self.crypto_key = app.config.get('GC_CRYPTO_KEY') - self.key_name = self.crypto_key_path(self.project_id, self.location, self.key_ring, self.crypto_key) - app.logger.info(f'Project ID: {self.project_id}') - app.logger.info(f'Location: {self.location}') - app.logger.info(f'Key Ring: {self.key_ring}') - app.logger.info(f'Crypto Key: {self.crypto_key}') - app.logger.info(f'Key Name: {self.key_name}') - - app.logger.info(f'Service Account Key Path: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}') - - os.environ["GOOGLE_CLOUD_PROJECT"] = self.project_id - - def encrypt_api_key(self, api_key): - """Encrypts the API key using the latest version of the KEK.""" - dek = get_random_bytes(32) # AES 256-bit key - cipher = AES.new(dek, AES.MODE_GCM) - ciphertext, tag = cipher.encrypt_and_digest(api_key.encode()) - # print(f'Dek: {dek}') - - # Encrypt the DEK using the latest version of the Google Cloud KMS key - encrypt_response = self.encrypt( - request={'name': self.key_name, 'plaintext': dek} - ) - encrypted_dek = encrypt_response.ciphertext - # print(f"Encrypted DEK: {encrypted_dek}") - # - # # Check - # decrypt_response = self.decrypt( - # request={'name': self.key_name, 'ciphertext': encrypted_dek} - # ) - # decrypted_dek = decrypt_response.plaintext - # print(f"Decrypted DEK: {decrypted_dek}") - - # Store the version of the key used - key_version = encrypt_response.name - - return { - 'key_version': key_version, - 'encrypted_dek': b64encode(encrypted_dek).decode('utf-8'), - 'nonce': b64encode(cipher.nonce).decode('utf-8'), - 'tag': b64encode(tag).decode('utf-8'), - 'ciphertext': b64encode(ciphertext).decode('utf-8') - } - - def decrypt_api_key(self, encrypted_data): - """Decrypts the API key using the specified key version.""" - if isinstance(encrypted_data, str): - encrypted_data = ast.literal_eval(encrypted_data) - key_version = encrypted_data['key_version'] - key_name = self.key_name - encrypted_dek = b64decode(encrypted_data['encrypted_dek'].encode('utf-8')) - nonce = b64decode(encrypted_data['nonce'].encode('utf-8')) - tag = b64decode(encrypted_data['tag'].encode('utf-8')) - ciphertext = b64decode(encrypted_data['ciphertext'].encode('utf-8')) - - # Decrypt the DEK using the specified version of the Google Cloud KMS key - try: - decrypt_response = self.decrypt( - request={'name': key_name, 'ciphertext': encrypted_dek} - ) - dek = decrypt_response.plaintext - except Exception as e: - print(f"Failed to decrypt DEK: {e}") - return None - - cipher = AES.new(dek, AES.MODE_GCM, nonce=nonce) - api_key = cipher.decrypt_and_verify(ciphertext, tag) - return api_key.decode() - - def check_kms_access_and_latency(self): - # key_name = self.crypto_key_path(self.project_id, self.location, self.key_ring, self.crypto_key) - # - # start_time = time.time() - # try: - # response = self.get_crypto_key(name=key_name) - # end_time = time.time() - # print(f"Response Time: {end_time - start_time} seconds") - # print("Access to KMS is successful.") - # except Exception as e: - # print(f"Failed to access KMS: {e}") - pass diff --git a/common/utils/simple_encryption.py b/common/utils/simple_encryption.py index d3ec947..5cfc51b 100644 --- a/common/utils/simple_encryption.py +++ b/common/utils/simple_encryption.py @@ -1,7 +1,13 @@ +import random from cryptography.fernet import Fernet from flask import Flask +def generate_api_key(prefix="EveAI-Chat"): + parts = [str(random.randint(1000, 9999)) for _ in range(5)] + return f"{prefix}-{'-'.join(parts)}" + + class SimpleEncryption: def __init__(self, app: Flask = None): self.app = app diff --git a/config/config.py b/config/config.py index f9f3ab6..b7e15b2 100644 --- a/config/config.py +++ b/config/config.py @@ -125,7 +125,6 @@ class DevConfig(Config): DEVELOPMENT = True DEBUG = True FLASK_DEBUG = True - PYCHARM_DEBUG = False EXPLAIN_TEMPLATE_LOADING = False # Database Settings @@ -191,6 +190,7 @@ class DevConfig(Config): MINIO_ENDPOINT = 'minio:9000' MINIO_ACCESS_KEY = 'minioadmin' MINIO_SECRET_KEY = 'minioadmin' + MINIO_USE_HTTPS = False class ProdConfig(Config): @@ -198,7 +198,6 @@ class ProdConfig(Config): DEBUG = False DEBUG = False FLASK_DEBUG = False - PYCHARM_DEBUG = False EXPLAIN_TEMPLATE_LOADING = False # Database Settings @@ -267,6 +266,12 @@ class ProdConfig(Config): # PATH settings ffmpeg_path = '/usr/bin/ffmpeg' + # MINIO + MINIO_ENDPOINT = environ.get('MINIO_ENDPOINT') + MINIO_ACCESS_KEY = environ.get('MINIO_ACCESS_KEY') + MINIO_SECRET_KEY = environ.get('MINIO_SECRET_KEY') + MINIO_USE_HTTPS = True + def get_config(config_name='dev'): configs = { diff --git a/config/logging_config.py b/config/logging_config.py index 60fb4c2..00ac088 100644 --- a/config/logging_config.py +++ b/config/logging_config.py @@ -1,3 +1,29 @@ +import os +from graypy import GELFUDPHandler +import logging +import logging.config + +# Graylog configuration +GRAYLOG_HOST = os.environ.get('GRAYLOG_HOST', 'localhost') +GRAYLOG_PORT = int(os.environ.get('GRAYLOG_PORT', 12201)) +env = os.environ.get('FLASK_ENV', 'development') + + +class CustomLogRecord(logging.LogRecord): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.component = os.environ.get('COMPONENT_NAME', 'eveai_app') # Set default component value here + + +def custom_log_record_factory(*args, **kwargs): + record = CustomLogRecord(*args, **kwargs) + return record + + +# Set the custom log record factory +logging.setLogRecordFactory(custom_log_record_factory) + + LOGGING = { 'version': 1, 'disable_existing_loggers': False, @@ -6,7 +32,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_app.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -14,7 +40,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_workers.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -22,7 +48,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_chat.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -30,7 +56,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/eveai_chat_workers.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -38,7 +64,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/sqlalchemy.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -46,7 +72,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/mailman.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -54,7 +80,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/security.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -62,7 +88,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/rag_tuning.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -70,7 +96,7 @@ LOGGING = { 'level': 'DEBUG', 'class': 'logging.handlers.RotatingFileHandler', 'filename': 'logs/embed_tuning.log', - 'maxBytes': 1024*1024*5, # 5MB + 'maxBytes': 1024 * 1024 * 5, # 5MB 'backupCount': 10, 'formatter': 'standard', }, @@ -79,55 +105,69 @@ LOGGING = { 'level': 'DEBUG', 'formatter': 'standard', }, + 'graylog': { + 'level': 'DEBUG', + 'class': 'graypy.GELFUDPHandler', + 'host': GRAYLOG_HOST, + 'port': GRAYLOG_PORT, + 'debugging_fields': True, # Set to True if you want to include debugging fields + 'extra_fields': True, # Set to True if you want to include extra fields + }, }, 'formatters': { 'standard': { - 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' + 'format': '%(asctime)s [%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' + '[Thread: %(threadName)s] [Host: %(hostname)s]: %(message)s' + }, + 'graylog': { + 'format': '[%(levelname)s] %(name)s (%(component)s) [%(module)s:%(lineno)d in %(funcName)s] ' + '[Thread: %(threadName)s] [Host: %(hostname)s]: %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S', }, }, 'loggers': { 'eveai_app': { # logger for the eveai_app - 'handlers': ['file_app',], + 'handlers': ['file_app', 'graylog', ] if env == 'production' else ['file_app', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_workers': { # logger for the eveai_workers - 'handlers': ['file_workers',], + 'handlers': ['file_workers', 'graylog', ] if env == 'production' else ['file_workers', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_chat': { # logger for the eveai_chat - 'handlers': ['file_chat',], + 'handlers': ['file_chat', 'graylog', ] if env == 'production' else ['file_chat', ], 'level': 'DEBUG', 'propagate': False }, 'eveai_chat_workers': { # logger for the eveai_chat_workers - 'handlers': ['file_chat_workers',], + 'handlers': ['file_chat_workers', 'graylog', ] if env == 'production' else ['file_chat_workers', ], 'level': 'DEBUG', 'propagate': False }, 'sqlalchemy.engine': { # logger for the sqlalchemy - 'handlers': ['file_sqlalchemy',], + 'handlers': ['file_sqlalchemy', 'graylog', ] if env == 'production' else ['file_sqlalchemy', ], 'level': 'DEBUG', 'propagate': False }, 'mailman': { # logger for the mailman - 'handlers': ['file_mailman', 'console'], + 'handlers': ['file_mailman', 'graylog', ] if env == 'production' else ['file_mailman', ], 'level': 'DEBUG', 'propagate': False }, 'security': { # logger for the security - 'handlers': ['file_security', 'console'], + 'handlers': ['file_security', 'graylog', ] if env == 'production' else ['file_security', ], 'level': 'DEBUG', 'propagate': False }, 'rag_tuning': { # logger for the rag_tuning - 'handlers': ['file_rag_tuning', 'console'], + 'handlers': ['file_rag_tuning', 'graylog', ] if env == 'production' else ['file_rag_tuning', ], 'level': 'DEBUG', 'propagate': False }, 'embed_tuning': { # logger for the embed_tuning - 'handlers': ['file_embed_tuning', 'console'], + 'handlers': ['file_embed_tuning', 'graylog', ] if env == 'production' else ['file_embed_tuning', ], 'level': 'DEBUG', 'propagate': False }, @@ -137,4 +177,4 @@ LOGGING = { 'propagate': False }, } -} \ No newline at end of file +} diff --git a/docker/build_and_push_eveai.sh b/docker/build_and_push_eveai.sh index a41cc86..fd49adc 100755 --- a/docker/build_and_push_eveai.sh +++ b/docker/build_and_push_eveai.sh @@ -3,6 +3,8 @@ # Exit on any error set -e +. ./docker_env_switch.sh dev + # Load environment variables source .env @@ -15,10 +17,64 @@ TAG="latest" # Platforms to build for PLATFORMS="linux/amd64,linux/arm64" -# Function to build and push a service -build_and_push_service() { +# Default action +ACTION="both" + +# Default build options +NO_CACHE="" +PROGRESS="" +DEBUG="" + +# Function to display usage information +usage() { + echo "Usage: $0 [-b|-p] [--no-cache] [--progress=plain] [--debug] [service1 service2 ...]" + echo " -b: Build only (for current platform)" + echo " -p: Push only (multi-platform)" + echo " --no-cache: Perform a clean build without using cache" + echo " --progress=plain: Show detailed progress of the build" + echo " --debug: Enable debug mode for the build" + echo " If no option is provided, both build and push will be performed." + echo " If no services are specified, all eveai_ services and nginx will be processed." +} + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case $1 in + -b) + ACTION="build" + shift + ;; + -p) + ACTION="push" + shift + ;; + --no-cache) + NO_CACHE="--no-cache" + shift + ;; + --progress=plain) + PROGRESS="--progress=plain" + shift + ;; + --debug) + DEBUG="--debug" + shift + ;; + -*) + echo "Unknown option: $1" + usage + exit 1 + ;; + *) + break + ;; + esac +done + +# Function to build and/or push a service +process_service() { local SERVICE="$1" - echo "Building and pushing $SERVICE..." + echo "Processing $SERVICE..." # Extract the build context and dockerfile from the compose file CONTEXT=$(yq e ".services.$SERVICE.build.context" compose_dev.yaml) @@ -36,18 +92,56 @@ build_and_push_service() { return 1 fi - # Build and push - docker buildx build \ - --platform "$PLATFORMS" \ - -t "$REGISTRY/$SERVICE:$TAG" \ - -f "$CONTEXT/$DOCKERFILE" \ - "$CONTEXT" \ - --push + # Build and/or push based on ACTION + if [ "$ACTION" = "build" ]; then + echo "Building $SERVICE for current platform..." + docker build \ + $NO_CACHE \ + $PROGRESS \ + $DEBUG \ + -t "$REGISTRY/$SERVICE:$TAG" \ + -f "$CONTEXT/$DOCKERFILE" \ + "$CONTEXT" + elif [ "$ACTION" = "push" ]; then + echo "Building and pushing $SERVICE for multiple platforms..." + docker buildx build \ + $NO_CACHE \ + $PROGRESS \ + $DEBUG \ + --platform "$PLATFORMS" \ + -t "$REGISTRY/$SERVICE:$TAG" \ + -f "$CONTEXT/$DOCKERFILE" \ + "$CONTEXT" \ + --push + else + echo "Building $SERVICE for current platform..." + docker build \ + $NO_CACHE \ + $PROGRESS \ + $DEBUG \ + -t "$REGISTRY/$SERVICE:$TAG" \ + -f "$CONTEXT/$DOCKERFILE" \ + "$CONTEXT" + + echo "Building and pushing $SERVICE for multiple platforms..." + docker buildx build \ + $NO_CACHE \ + $PROGRESS \ + $DEBUG \ + --platform "$PLATFORMS" \ + -t "$REGISTRY/$SERVICE:$TAG" \ + -f "$CONTEXT/$DOCKERFILE" \ + "$CONTEXT" \ + --push + fi } -# If no arguments are provided, build all services +# If no arguments are provided, process all services if [ $# -eq 0 ]; then - mapfile -t SERVICES < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)') + SERVICES=() + while IFS= read -r line; do + SERVICES+=("$line") + done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)') else SERVICES=("$@") fi @@ -65,10 +159,10 @@ docker buildx use eveai_builder # Loop through services for SERVICE in "${SERVICES[@]}"; do if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* ]]; then - if build_and_push_service "$SERVICE"; then - echo "Successfully built and pushed $SERVICE" + if process_service "$SERVICE"; then + echo "Successfully processed $SERVICE" else - echo "Failed to build and push $SERVICE" + echo "Failed to process $SERVICE" fi else echo "Skipping $SERVICE as it's not nginx or doesn't start with eveai_" diff --git a/docker/compose_dev.yaml b/docker/compose_dev.yaml index 2dff6e2..3e2b28f 100644 --- a/docker/compose_dev.yaml +++ b/docker/compose_dev.yaml @@ -29,9 +29,16 @@ x-common-variables: &common-variables MINIO_ENDPOINT: minio:9000 MINIO_ACCESS_KEY: minioadmin MINIO_SECRET_KEY: minioadmin + NGINX_SERVER_NAME: 'localhost http://macstudio.ask-eve-ai-local.com/' + + +networks: + eveai-network: + driver: bridge services: nginx: + image: josakola/nginx:latest build: context: .. dockerfile: ./docker/nginx/Dockerfile @@ -41,6 +48,8 @@ services: ports: - 80:80 - 8080:8080 + environment: + <<: *common-variables volumes: - ../nginx:/etc/nginx - ../nginx/sites-enabled:/etc/nginx/sites-enabled @@ -50,8 +59,11 @@ services: depends_on: - eveai_app - eveai_chat + networks: + - eveai-network eveai_app: + image: josakola/eveai_app:latest build: context: .. dockerfile: ./docker/eveai_app/Dockerfile @@ -62,6 +74,7 @@ services: - 5001:5001 environment: <<: *common-variables + COMPONENT_NAME: eveai_app volumes: - ../eveai_app:/app/eveai_app - ../common:/app/common @@ -69,7 +82,7 @@ services: - ../migrations:/app/migrations - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - ./logs:/app/logs + - eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -82,9 +95,13 @@ services: interval: 10s timeout: 5s retries: 5 - command: ["sh", "-c", "scripts/start_eveai_app.sh"] +# entrypoint: ["scripts/entrypoint.sh"] +# command: ["scripts/start_eveai_app.sh"] + networks: + - eveai-network eveai_workers: + image: josakola/eveai_workers:latest build: context: .. dockerfile: ./docker/eveai_workers/Dockerfile @@ -95,13 +112,14 @@ services: # - 5001:5001 environment: <<: *common-variables + COMPONENT_NAME: eveai_workers volumes: - ../eveai_workers:/app/eveai_workers - ../common:/app/common - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - ./logs:/app/logs + - eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -114,9 +132,13 @@ services: # interval: 10s # timeout: 5s # retries: 5 - command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ] +# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ] +# command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ] + networks: + - eveai-network eveai_chat: + image: josakola/eveai_chat:latest build: context: .. dockerfile: ./docker/eveai_chat/Dockerfile @@ -127,13 +149,14 @@ services: - 5002:5002 environment: <<: *common-variables + COMPONENT_NAME: eveai_chat volumes: - ../eveai_chat:/app/eveai_chat - ../common:/app/common - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - ./logs:/app/logs + - eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -144,9 +167,13 @@ services: interval: 10s timeout: 5s retries: 5 - command: ["sh", "-c", "scripts/start_eveai_chat.sh"] +# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ] +# command: ["sh", "-c", "scripts/start_eveai_chat.sh"] + networks: + - eveai-network eveai_chat_workers: + image: josakola/eveai_chat_workers:latest build: context: .. dockerfile: ./docker/eveai_chat_workers/Dockerfile @@ -157,13 +184,14 @@ services: # - 5001:5001 environment: <<: *common-variables + COMPONENT_NAME: eveai_chat_workers volumes: - ../eveai_chat_workers:/app/eveai_chat_workers - ../common:/app/common - ../config:/app/config - ../scripts:/app/scripts - ../patched_packages:/app/patched_packages - - ./logs:/app/logs + - eveai_logs:/app/logs depends_on: db: condition: service_healthy @@ -174,7 +202,10 @@ services: # interval: 10s # timeout: 5s # retries: 5 - command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ] +# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ] +# command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ] + networks: + - eveai-network db: hostname: db @@ -194,6 +225,8 @@ services: interval: 10s timeout: 5s retries: 5 + networks: + - eveai-network redis: image: redis:7.2.5 @@ -207,6 +240,8 @@ services: interval: 10s timeout: 5s retries: 5 + networks: + - eveai-network minio: image: minio/minio @@ -228,9 +263,12 @@ services: timeout: 20s retries: 3 start_period: 30s + networks: + - eveai-network volumes: minio_data: + eveai_logs: # db-data: # redis-data: # tenant-files: diff --git a/docker/compose_stackhero.yaml b/docker/compose_stackhero.yaml index 240b0a5..86edbaa 100644 --- a/docker/compose_stackhero.yaml +++ b/docker/compose_stackhero.yaml @@ -20,16 +20,26 @@ x-common-variables: &common-variables SECURITY_PASSWORD_SALT: '166448071751628781809462050022558634074' MAIL_USERNAME: 'evie_admin@askeveai.com' MAIL_PASSWORD: 's5D%R#y^v!s&6Z^i0k&' - REDIS_USER: admin - REDIS_PASS: 'b32vtDtLriSY1fL2zGrZg8IZKI0g9ucsLtVNanRFAras6oZ51wjVNB1Y05uG7uEw' + REDIS_USER: eveai + REDIS_PASS: 'jHliZwGD36sONgbm0fc6SOpzLbknqq4RNF8K' REDIS_URL: 8bciqc.stackhero-network.com REDIS_PORT: '9961' OPENAI_API_KEY: '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' GROQ_API_KEY: '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' ANTHROPIC_API_KEY: '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' - PORTKEY_API_KEY: '3C+zAGR8pCalevBXFVc0l8R2MPYc' + PORTKEY_API_KEY: 'XvmvBFIVbm76opUxA7MNP14QmdQj' JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408' API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0=' + GRAYLOG_HOST: de4zvu.stackhero-network.com + GRAYLOG_PORT: '12201' + MINIO_ENDPOINT: 'fxwnyl.stackhero-network.com:443' + MINIO_ACCESS_KEY: 04JKmQln8PQpyTmMiCPc + MINIO_SECRET_KEY: 2PEZAD1nlpAmOyDV0TUTuJTQw1qVuYLF3A7GMs0D + NGINX_SERVER_NAME: 'evie.askeveai.com mxz536.stackhero-network.com' + +networks: + eveai-network: + driver: bridge services: nginx: @@ -38,19 +48,25 @@ services: ports: - 80:80 - 8080:8080 + environment: + <<: *common-variables volumes: # - ../nginx:/etc/nginx # - ../nginx/sites-enabled:/etc/nginx/sites-enabled # - ../nginx/static:/etc/nginx/static # - ../nginx/public:/etc/nginx/public - - logs:/var/log/nginx + - eveai_logs:/var/log/nginx labels: - "traefik.enable=true" - - "traefik.http.routers.api.rule=Host(`evie.askeveai.com`)" + - "traefik.http.routers.nginx.rule=Host(`evie.askeveai.com`) || Host(`mxz536.stackhero-network.com`)" + - "traefik.http.routers.nginx.entrypoints=web,websecure" + - "traefik.http.routers.nginx.tls.certresolver=myresolver" - "traefik.http.services.nginx.loadbalancer.server.port=80" depends_on: - eveai_app - eveai_chat + networks: + - eveai-network eveai_app: platform: linux/amd64 @@ -59,14 +75,16 @@ services: - 5001:5001 environment: <<: *common-variables + COMPONENT_NAME: eveai_app volumes: - - logs:/app/logs + - eveai_logs:/app/logs healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5001/health"] interval: 10s timeout: 5s retries: 5 - command: ["sh", "-c", "scripts/start_eveai_app.sh"] + networks: + - eveai-network eveai_workers: platform: linux/amd64 @@ -75,14 +93,16 @@ services: # - 5001:5001 environment: <<: *common-variables + COMPONENT_NAME: eveai_workers volumes: - - logs:/app/logs + - eveai_logs:/app/logs # healthcheck: # test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ] # interval: 10s # timeout: 5s # retries: 5 - command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ] + networks: + - eveai-network eveai_chat: platform: linux/amd64 @@ -91,14 +111,16 @@ services: - 5002:5002 environment: <<: *common-variables + COMPONENT_NAME: eveai_chat volumes: - - logs:/app/logs + - eveai_logs:/app/logs healthcheck: test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ] # Adjust based on your health endpoint interval: 10s timeout: 5s retries: 5 - command: ["sh", "-c", "scripts/start_eveai_chat.sh"] + networks: + - eveai-network eveai_chat_workers: platform: linux/amd64 @@ -107,18 +129,19 @@ services: # - 5001:5001 environment: <<: *common-variables + COMPONENT_NAME: eveai_chat_workers volumes: - - logs:/app/logs + - eveai_logs:/app/logs # healthcheck: # test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ] # interval: 10s # timeout: 5s # retries: 5 - command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ] - + networks: + - eveai-network volumes: - logs: + eveai_logs: # miniAre theo_data: # db-data: # redis-data: diff --git a/docker/copy_docker_logs.sh b/docker/copy_docker_logs.sh new file mode 100755 index 0000000..a97481d --- /dev/null +++ b/docker/copy_docker_logs.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Check if a volume name was provided +if [ $# -eq 0 ]; then + echo "Error: No volume name provided." + echo "Usage: $0 " + exit 1 +fi + +# Get the volume name from the first argument +VOLUME_NAME=$1 + +# Define the local directory to copy logs to +LOCAL_DIR="./stackhero/logs" + +# Create the local directory if it doesn't exist +mkdir -p "$LOCAL_DIR" + +echo "Copying logs from Docker volume '$VOLUME_NAME' to $LOCAL_DIR" + +# Create a temporary container with the volume mounted +CONTAINER_ID=$(docker run -d -v $VOLUME_NAME:/logs --name temp_log_container alpine sh -c 'tail -f /dev/null') + +# Check if the container was created successfully +if [ $? -ne 0 ]; then + echo "Error: Failed to create temporary container." + exit 1 +fi + +# Copy files from the container to the local machine +docker cp $CONTAINER_ID:/logs/. "$LOCAL_DIR" + +# Check if the copy was successful +if [ $? -eq 0 ]; then + echo "Logs successfully copied to $LOCAL_DIR" +else + echo "Error: Failed to copy logs from the container." +fi + +# Remove the temporary container +docker rm -f $CONTAINER_ID + +echo "Temporary container removed." \ No newline at end of file diff --git a/docker/docker_env_switch.sh b/docker/docker_env_switch.sh index 3f3a70e..8dc8517 100755 --- a/docker/docker_env_switch.sh +++ b/docker/docker_env_switch.sh @@ -48,10 +48,12 @@ echo "Set COMPOSE_FILE to $COMPOSE_FILE" # Define aliases for common Docker commands alias docker-compose="docker compose -f $COMPOSE_FILE" alias dc="docker compose -f $COMPOSE_FILE" -alias dcup="docker compose -f $COMPOSE_FILE up -d" +alias dcup="docker compose -f $COMPOSE_FILE up -d --no-build --remove-orphans" alias dcdown="docker compose -f $COMPOSE_FILE down" alias dcps="docker compose -f $COMPOSE_FILE ps" alias dclogs="docker compose -f $COMPOSE_FILE logs" +alias dcpull="docker compose -f $COMPOSE_FILE pull" +alias dcrefresh="docker compose -f $COMPOSE_FILE pull" echo "Docker environment switched to $1" -echo "You can now use 'docker-compose', 'dc', 'dcup', 'dcdown', 'dcps', and 'dclogs' commands" \ No newline at end of file +echo "You can now use 'docker-compose', 'dc', 'dcup', 'dcdown', 'dcps', 'dclogs' , dcpull or dcrefresh commands" \ No newline at end of file diff --git a/docker/eveai_app/Dockerfile b/docker/eveai_app/Dockerfile index 0b4159e..c7fc9dd 100644 --- a/docker/eveai_app/Dockerfile +++ b/docker/eveai_app/Dockerfile @@ -24,7 +24,7 @@ RUN adduser \ --disabled-password \ --gecos "" \ --home "/nonexistent" \ - --shell "/sbin/nologin" \ + --shell "/bin/bash" \ --no-create-home \ --uid "${UID}" \ appuser @@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Create logs directory and set permissions +RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs + # Download dependencies as a separate step to take advantage of Docker's caching. # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds. # Leverage a bind mount to requirements.txt to avoid having to copy them into @@ -45,9 +48,6 @@ RUN apt-get update && apt-get install -y \ COPY requirements.txt /app/ RUN python -m pip install -r /app/requirements.txt -# Switch to the non-privileged user to run the application. -USER appuser - # Copy the source code into the container. COPY eveai_app /app/eveai_app COPY common /app/common @@ -56,8 +56,15 @@ COPY migrations /app/migrations COPY scripts /app/scripts COPY patched_packages /app/patched_packages +# Set permissions for entrypoint script +RUN chmod 777 /app/scripts/entrypoint.sh + +# Set ownership of the application directory to the non-privileged user +RUN chown -R appuser:appuser /app + # Expose the port that the application listens on. EXPOSE 5001 -# Run the application. -#CMD ["sh", "-c", "/app/scripts/start_eveai_app.sh"] +# Set entrypoint and command +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["/app/scripts/start_eveai_app.sh"] diff --git a/docker/eveai_chat/Dockerfile b/docker/eveai_chat/Dockerfile index 9e3bcc7..6b694b7 100644 --- a/docker/eveai_chat/Dockerfile +++ b/docker/eveai_chat/Dockerfile @@ -24,7 +24,7 @@ RUN adduser \ --disabled-password \ --gecos "" \ --home "/nonexistent" \ - --shell "/sbin/nologin" \ + --shell "/bin/bash" \ --no-create-home \ --uid "${UID}" \ appuser @@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Create logs directory and set permissions +RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs + # Download dependencies as a separate step to take advantage of Docker's caching. # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds. # Leverage a bind mount to requirements.txt to avoid having to copy them into @@ -45,9 +48,6 @@ RUN apt-get update && apt-get install -y \ COPY ../../requirements.txt /app/ RUN python -m pip install -r requirements.txt -# Switch to the non-privileged user to run the application. -USER appuser - # Copy the source code into the container. COPY eveai_chat /app/eveai_chat COPY common /app/common @@ -55,8 +55,15 @@ COPY config /app/config COPY scripts /app/scripts COPY patched_packages /app/patched_packages +# Set permissions for entrypoint script +RUN chmod 777 /app/scripts/entrypoint.sh + +# Set ownership of the application directory to the non-privileged user +RUN chown -R appuser:appuser /app + # Expose the port that the application listens on. EXPOSE 5002 -# Run the application. -#CMD ["sh", "-c", "/app/scripts/start_eveai_chat.sh"] +# Set entrypoint and command +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["/app/scripts/start_eveai_chat.sh"] diff --git a/docker/eveai_chat_workers/Dockerfile b/docker/eveai_chat_workers/Dockerfile index ffa573d..b485969 100644 --- a/docker/eveai_chat_workers/Dockerfile +++ b/docker/eveai_chat_workers/Dockerfile @@ -24,7 +24,7 @@ RUN adduser \ --disabled-password \ --gecos "" \ --home "/nonexistent" \ - --shell "/sbin/nologin" \ + --shell "/bin/bash" \ --no-create-home \ --uid "${UID}" \ appuser @@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Create logs directory and set permissions +RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs + # Download dependencies as a separate step to take advantage of Docker's caching. # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds. # Leverage a bind mount to requirements.txt to avoid having to copy them into @@ -45,18 +48,20 @@ RUN apt-get update && apt-get install -y \ COPY requirements.txt /app/ RUN python -m pip install -r /app/requirements.txt -# Switch to the non-privileged user to run the application. -USER appuser - # Copy the source code into the container. COPY eveai_chat_workers /app/eveai_chat_workers COPY common /app/common COPY config /app/config COPY scripts /app/scripts COPY patched_packages /app/patched_packages +COPY --chown=root:root scripts/entrypoint.sh /app/scripts/ -# Expose the port that the application listens on. -#EXPOSE 5001 +# Set permissions for entrypoint script +RUN chmod 777 /app/scripts/entrypoint.sh -# Run the application. -# CMD ["sh", "-c", "/app/scripts/start_eveai_workers.sh"] +# Set ownership of the application directory to the non-privileged user +RUN chown -R appuser:appuser /app + +# Set entrypoint and command +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["/app/scripts/start_eveai_chat_workers.sh"] diff --git a/docker/eveai_workers/Dockerfile b/docker/eveai_workers/Dockerfile index 8e71fba..8f32738 100644 --- a/docker/eveai_workers/Dockerfile +++ b/docker/eveai_workers/Dockerfile @@ -24,7 +24,7 @@ RUN adduser \ --disabled-password \ --gecos "" \ --home "/nonexistent" \ - --shell "/sbin/nologin" \ + --shell "/bin/bash" \ --no-create-home \ --uid "${UID}" \ appuser @@ -38,6 +38,9 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Create logs directory and set permissions +RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs + # Install Python dependencies. # Download dependencies as a separate step to take advantage of Docker's caching. @@ -48,18 +51,20 @@ RUN apt-get update && apt-get install -y \ COPY requirements.txt /app/ RUN python -m pip install -r /app/requirements.txt -# Switch to the non-privileged user to run the application. -USER appuser - # Copy the source code into the container. COPY eveai_workers /app/eveai_workers COPY common /app/common COPY config /app/config COPY scripts /app/scripts COPY patched_packages /app/patched_packages +COPY --chown=root:root scripts/entrypoint.sh /app/scripts/ -# Expose the port that the application listens on. -#EXPOSE 5001 +# Set permissions for entrypoint script +RUN chmod 777 /app/scripts/entrypoint.sh -# Run the application. -# CMD ["sh", "-c", "/app/scripts/start_eveai_workers.sh"] +# Set ownership of the application directory to the non-privileged user +RUN chown -R appuser:appuser /app + +# Set entrypoint and command +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["/app/scripts/start_eveai_workers.sh"] diff --git a/eveai_app/views/user_views.py b/eveai_app/views/user_views.py index fcf70e6..296b49e 100644 --- a/eveai_app/views/user_views.py +++ b/eveai_app/views/user_views.py @@ -13,7 +13,7 @@ from common.utils.security_utils import send_confirmation_email, send_reset_emai from .user_forms import TenantForm, CreateUserForm, EditUserForm, TenantDomainForm from common.utils.database import Database from common.utils.view_assistants import prepare_table_for_macro, form_validation_failed -from common.utils.key_encryption import generate_api_key +from common.utils.simple_encryption import generate_api_key from common.utils.nginx_utils import prefixed_url_for user_bp = Blueprint('user_bp', __name__, url_prefix='/user') diff --git a/eveai_workers/tasks.py b/eveai_workers/tasks.py index 0c76f7a..d90b0ca 100644 --- a/eveai_workers/tasks.py +++ b/eveai_workers/tasks.py @@ -275,14 +275,16 @@ def summarize_chunk(tenant, model_variables, document_version, chunk): llm = model_variables['llm'] template = model_variables['summary_template'] language_template = create_language_template(template, document_version.language) - current_app.logger.debug(f'Language prompt: {language_template}') - chain = load_summarize_chain(llm, chain_type='stuff', prompt=ChatPromptTemplate.from_template(language_template)) + summary_prompt = ChatPromptTemplate.from_template(language_template) + setup = RunnablePassthrough() + output_parser = StrOutputParser() - doc_creator = CharacterTextSplitter(chunk_size=model_variables['max_chunk_size'] * 2, chunk_overlap=0) - text_to_summarize = doc_creator.create_documents(chunk) + chain = setup | summary_prompt | llm | output_parser try: - summary = chain.invoke({"text": text_to_summarize}) + current_app.logger.debug(f'Starting summarizing chunk for tenant {tenant.id} ' + f'on document version {document_version.id}') + summary = chain.invoke({"text": chunk}) current_app.logger.debug(f'Finished summarizing chunk for tenant {tenant.id} ' f'on document version {document_version.id}.') return summary diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 82ed99a..c28dfe0 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -42,7 +42,7 @@ http { server { listen 80; listen 8080; - server_name localhost macstudio.ask-eve-ai-local.com; + server_name ${NGINX_SERVER_NAME}; #charset koi8-r; diff --git a/requirements.txt b/requirements.txt index 9001be0..c1b966c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,12 +22,6 @@ Flask-SQLAlchemy~=3.1.1 Flask-WTF~=1.2.1 gevent~=24.2.1 gevent-websocket~=0.10.1 -google~=3.0.0 -google-api-core~=2.19.1rc0 -google-auth~=2.30.0 -google-cloud-core~=2.4.1 -google-cloud-kms~=2.23.0 -googleapis-common-protos~=1.63.2rc0 greenlet~=3.0.3 gunicorn~=22.0.0 Jinja2~=3.1.4 @@ -48,7 +42,6 @@ pg8000~=1.31.2 pgvector~=0.2.5 pycryptodome~=3.20.0 pydantic~=2.7.4 -pydevd-pycharm~=242.18071.12 PyJWT~=2.8.0 pypdf~=4.2.0 PySocks~=1.7.1 @@ -79,4 +72,5 @@ portkey_ai~=1.7.0 minio~=7.2.7 Werkzeug~=3.0.3 itsdangerous~=2.2.0 -cryptography~=43.0.0 \ No newline at end of file +cryptography~=43.0.0 +graypy~=2.1.0 \ No newline at end of file diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..ccf2e99 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +# Ensure the logs directory has the correct permissions +echo "Changing permissions on logs directory" +#chown -R appuser:appuser /app/logs +chmod -R 777 /app/logs + +# Wait for the database to be ready +until pg_isready -h $DB_HOST -p $DB_PORT; do + echo "Postgres is unavailable - sleeping" + sleep 2 +done + +# Switch to appuser and execute the command passed to the script +exec su appuser -c "$@" diff --git a/scripts/start_eveai_app.sh b/scripts/start_eveai_app.sh index ec3142f..541783b 100755 --- a/scripts/start_eveai_app.sh +++ b/scripts/start_eveai_app.sh @@ -3,20 +3,23 @@ cd "/app" || exit 1 export PYTHONPATH="$PYTHONPATH:/app/" +# Ensure we can write the logs +chown -R appuser:appuser /app/logs + # Wait for the database to be ready echo "Waiting for database to be ready" -until pg_isready -h db -p 5432; do +until pg_isready -h $DB_HOST -p $DB_PORT; do echo "Postgres is unavailable - sleeping" sleep 2 done echo "Postgres is up - executing commands" -export PGPASSWORD=Skywalker! +export PGPASSWORD=$DB_PASS # Check if the database exists and initialize if not -if ! psql -U luke -h db -d eveai -c '\dt' | grep -q 'No relations found'; then +if ! psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d $DB_NAME -c '\dt' | grep -q 'No relations found'; then echo "Database eveai does not exist or is empty. Initializing..." - psql -U luke -h db -d postgres -c "CREATE DATABASE eveai;" - psql -U luke -h db -d eveai -c "CREATE EXTENSION IF NOT EXISTS vector;" + psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -c "CREATE DATABASE $DB_NAME;" + psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d $DB_NAME -c "CREATE EXTENSION IF NOT EXISTS vector;" fi echo "Applying migrations to the public and tenant schema..." diff --git a/scripts/start_eveai_chat.sh b/scripts/start_eveai_chat.sh index 789410a..7b69a4c 100755 --- a/scripts/start_eveai_chat.sh +++ b/scripts/start_eveai_chat.sh @@ -4,6 +4,9 @@ cd "/app/" || exit 1 export PROJECT_DIR="/app" export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR" # Include the app directory in the Python path & patched packages +# Ensure we can write the logs +chown -R appuser:appuser /app/logs + # Set flask environment variables #export FLASK_ENV=development # Use 'production' as appropriate #export FLASK_DEBUG=1 # Use 0 for production diff --git a/scripts/start_eveai_chat_workers.sh b/scripts/start_eveai_chat_workers.sh index 78f2e6a..b2c2cc2 100755 --- a/scripts/start_eveai_chat_workers.sh +++ b/scripts/start_eveai_chat_workers.sh @@ -4,6 +4,9 @@ cd "/app/" || exit 1 export PROJECT_DIR="/app" export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR" # Include the app directory in the Python path & patched packages +# Ensure we can write the logs +chown -R appuser:appuser /app/logs + # Start a worker for the 'llm_interactions' queue with auto-scaling celery -A eveai_chat_workers.celery worker --loglevel=info -Q llm_interactions --autoscale=2,8 --hostname=interactions_worker@%h & diff --git a/scripts/start_eveai_workers.sh b/scripts/start_eveai_workers.sh index 6ab608e..374db96 100755 --- a/scripts/start_eveai_workers.sh +++ b/scripts/start_eveai_workers.sh @@ -4,6 +4,9 @@ cd "/app/" || exit 1 export PROJECT_DIR="/app" export PYTHONPATH="$PROJECT_DIR/patched_packages:$PYTHONPATH:$PROJECT_DIR" # Include the app directory in the Python path & patched packages +# Ensure we can write the logs +chown -R appuser:appuser /app/logs + # Start a worker for the 'embeddings' queue with higher concurrency celery -A eveai_workers.celery worker --loglevel=info -Q embeddings --autoscale=2,8 --hostname=embeddings_worker@%h &