- Improvements working with the cloud, minio, graylog and some first bugfixing

This commit is contained in:
Josako
2024-08-13 09:04:19 +02:00
parent 9caa423bcf
commit ab38dd7540
23 changed files with 414 additions and 220 deletions

View File

@@ -3,6 +3,8 @@
# Exit on any error
set -e
. ./docker_env_switch.sh dev
# Load environment variables
source .env
@@ -15,10 +17,64 @@ TAG="latest"
# Platforms to build for
PLATFORMS="linux/amd64,linux/arm64"
# Function to build and push a service
build_and_push_service() {
# Default action
ACTION="both"
# Default build options
NO_CACHE=""
PROGRESS=""
DEBUG=""
# Function to display usage information
usage() {
echo "Usage: $0 [-b|-p] [--no-cache] [--progress=plain] [--debug] [service1 service2 ...]"
echo " -b: Build only (for current platform)"
echo " -p: Push only (multi-platform)"
echo " --no-cache: Perform a clean build without using cache"
echo " --progress=plain: Show detailed progress of the build"
echo " --debug: Enable debug mode for the build"
echo " If no option is provided, both build and push will be performed."
echo " If no services are specified, all eveai_ services and nginx will be processed."
}
# Parse command-line options
while [[ $# -gt 0 ]]; do
case $1 in
-b)
ACTION="build"
shift
;;
-p)
ACTION="push"
shift
;;
--no-cache)
NO_CACHE="--no-cache"
shift
;;
--progress=plain)
PROGRESS="--progress=plain"
shift
;;
--debug)
DEBUG="--debug"
shift
;;
-*)
echo "Unknown option: $1"
usage
exit 1
;;
*)
break
;;
esac
done
# Function to build and/or push a service
process_service() {
local SERVICE="$1"
echo "Building and pushing $SERVICE..."
echo "Processing $SERVICE..."
# Extract the build context and dockerfile from the compose file
CONTEXT=$(yq e ".services.$SERVICE.build.context" compose_dev.yaml)
@@ -36,18 +92,56 @@ build_and_push_service() {
return 1
fi
# Build and push
docker buildx build \
--platform "$PLATFORMS" \
-t "$REGISTRY/$SERVICE:$TAG" \
-f "$CONTEXT/$DOCKERFILE" \
"$CONTEXT" \
--push
# Build and/or push based on ACTION
if [ "$ACTION" = "build" ]; then
echo "Building $SERVICE for current platform..."
docker build \
$NO_CACHE \
$PROGRESS \
$DEBUG \
-t "$REGISTRY/$SERVICE:$TAG" \
-f "$CONTEXT/$DOCKERFILE" \
"$CONTEXT"
elif [ "$ACTION" = "push" ]; then
echo "Building and pushing $SERVICE for multiple platforms..."
docker buildx build \
$NO_CACHE \
$PROGRESS \
$DEBUG \
--platform "$PLATFORMS" \
-t "$REGISTRY/$SERVICE:$TAG" \
-f "$CONTEXT/$DOCKERFILE" \
"$CONTEXT" \
--push
else
echo "Building $SERVICE for current platform..."
docker build \
$NO_CACHE \
$PROGRESS \
$DEBUG \
-t "$REGISTRY/$SERVICE:$TAG" \
-f "$CONTEXT/$DOCKERFILE" \
"$CONTEXT"
echo "Building and pushing $SERVICE for multiple platforms..."
docker buildx build \
$NO_CACHE \
$PROGRESS \
$DEBUG \
--platform "$PLATFORMS" \
-t "$REGISTRY/$SERVICE:$TAG" \
-f "$CONTEXT/$DOCKERFILE" \
"$CONTEXT" \
--push
fi
}
# If no arguments are provided, build all services
# If no arguments are provided, process all services
if [ $# -eq 0 ]; then
mapfile -t SERVICES < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)')
SERVICES=()
while IFS= read -r line; do
SERVICES+=("$line")
done < <(yq e '.services | keys | .[]' compose_dev.yaml | grep -E '^(nginx|eveai_)')
else
SERVICES=("$@")
fi
@@ -65,10 +159,10 @@ docker buildx use eveai_builder
# Loop through services
for SERVICE in "${SERVICES[@]}"; do
if [[ "$SERVICE" == "nginx" || "$SERVICE" == eveai_* ]]; then
if build_and_push_service "$SERVICE"; then
echo "Successfully built and pushed $SERVICE"
if process_service "$SERVICE"; then
echo "Successfully processed $SERVICE"
else
echo "Failed to build and push $SERVICE"
echo "Failed to process $SERVICE"
fi
else
echo "Skipping $SERVICE as it's not nginx or doesn't start with eveai_"

View File

@@ -29,9 +29,16 @@ x-common-variables: &common-variables
MINIO_ENDPOINT: minio:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
NGINX_SERVER_NAME: 'localhost http://macstudio.ask-eve-ai-local.com/'
networks:
eveai-network:
driver: bridge
services:
nginx:
image: josakola/nginx:latest
build:
context: ..
dockerfile: ./docker/nginx/Dockerfile
@@ -41,6 +48,8 @@ services:
ports:
- 80:80
- 8080:8080
environment:
<<: *common-variables
volumes:
- ../nginx:/etc/nginx
- ../nginx/sites-enabled:/etc/nginx/sites-enabled
@@ -50,8 +59,11 @@ services:
depends_on:
- eveai_app
- eveai_chat
networks:
- eveai-network
eveai_app:
image: josakola/eveai_app:latest
build:
context: ..
dockerfile: ./docker/eveai_app/Dockerfile
@@ -62,6 +74,7 @@ services:
- 5001:5001
environment:
<<: *common-variables
COMPONENT_NAME: eveai_app
volumes:
- ../eveai_app:/app/eveai_app
- ../common:/app/common
@@ -69,7 +82,7 @@ services:
- ../migrations:/app/migrations
- ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages
- ./logs:/app/logs
- eveai_logs:/app/logs
depends_on:
db:
condition: service_healthy
@@ -82,9 +95,13 @@ services:
interval: 10s
timeout: 5s
retries: 5
command: ["sh", "-c", "scripts/start_eveai_app.sh"]
# entrypoint: ["scripts/entrypoint.sh"]
# command: ["scripts/start_eveai_app.sh"]
networks:
- eveai-network
eveai_workers:
image: josakola/eveai_workers:latest
build:
context: ..
dockerfile: ./docker/eveai_workers/Dockerfile
@@ -95,13 +112,14 @@ services:
# - 5001:5001
environment:
<<: *common-variables
COMPONENT_NAME: eveai_workers
volumes:
- ../eveai_workers:/app/eveai_workers
- ../common:/app/common
- ../config:/app/config
- ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages
- ./logs:/app/logs
- eveai_logs:/app/logs
depends_on:
db:
condition: service_healthy
@@ -114,9 +132,13 @@ services:
# interval: 10s
# timeout: 5s
# retries: 5
command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
# command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
networks:
- eveai-network
eveai_chat:
image: josakola/eveai_chat:latest
build:
context: ..
dockerfile: ./docker/eveai_chat/Dockerfile
@@ -127,13 +149,14 @@ services:
- 5002:5002
environment:
<<: *common-variables
COMPONENT_NAME: eveai_chat
volumes:
- ../eveai_chat:/app/eveai_chat
- ../common:/app/common
- ../config:/app/config
- ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages
- ./logs:/app/logs
- eveai_logs:/app/logs
depends_on:
db:
condition: service_healthy
@@ -144,9 +167,13 @@ services:
interval: 10s
timeout: 5s
retries: 5
command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
# command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
networks:
- eveai-network
eveai_chat_workers:
image: josakola/eveai_chat_workers:latest
build:
context: ..
dockerfile: ./docker/eveai_chat_workers/Dockerfile
@@ -157,13 +184,14 @@ services:
# - 5001:5001
environment:
<<: *common-variables
COMPONENT_NAME: eveai_chat_workers
volumes:
- ../eveai_chat_workers:/app/eveai_chat_workers
- ../common:/app/common
- ../config:/app/config
- ../scripts:/app/scripts
- ../patched_packages:/app/patched_packages
- ./logs:/app/logs
- eveai_logs:/app/logs
depends_on:
db:
condition: service_healthy
@@ -174,7 +202,10 @@ services:
# interval: 10s
# timeout: 5s
# retries: 5
command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
# entrypoint: [ "sh", "-c", "scripts/entrypoint.sh" ]
# command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
networks:
- eveai-network
db:
hostname: db
@@ -194,6 +225,8 @@ services:
interval: 10s
timeout: 5s
retries: 5
networks:
- eveai-network
redis:
image: redis:7.2.5
@@ -207,6 +240,8 @@ services:
interval: 10s
timeout: 5s
retries: 5
networks:
- eveai-network
minio:
image: minio/minio
@@ -228,9 +263,12 @@ services:
timeout: 20s
retries: 3
start_period: 30s
networks:
- eveai-network
volumes:
minio_data:
eveai_logs:
# db-data:
# redis-data:
# tenant-files:

View File

@@ -20,16 +20,26 @@ x-common-variables: &common-variables
SECURITY_PASSWORD_SALT: '166448071751628781809462050022558634074'
MAIL_USERNAME: 'evie_admin@askeveai.com'
MAIL_PASSWORD: 's5D%R#y^v!s&6Z^i0k&'
REDIS_USER: admin
REDIS_PASS: 'b32vtDtLriSY1fL2zGrZg8IZKI0g9ucsLtVNanRFAras6oZ51wjVNB1Y05uG7uEw'
REDIS_USER: eveai
REDIS_PASS: 'jHliZwGD36sONgbm0fc6SOpzLbknqq4RNF8K'
REDIS_URL: 8bciqc.stackhero-network.com
REDIS_PORT: '9961'
OPENAI_API_KEY: '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'
GROQ_API_KEY: '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'
ANTHROPIC_API_KEY: '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'
PORTKEY_API_KEY: '3C+zAGR8pCalevBXFVc0l8R2MPYc'
PORTKEY_API_KEY: 'XvmvBFIVbm76opUxA7MNP14QmdQj'
JWT_SECRET_KEY: '0d99e810e686ea567ef305d8e9b06195c4db482952e19276590a726cde60a408'
API_ENCRYPTION_KEY: 'Ly5XYWwEKiasfAwEqdEMdwR-k0vhrq6QPYd4whEROB0='
GRAYLOG_HOST: de4zvu.stackhero-network.com
GRAYLOG_PORT: '12201'
MINIO_ENDPOINT: 'fxwnyl.stackhero-network.com:443'
MINIO_ACCESS_KEY: 04JKmQln8PQpyTmMiCPc
MINIO_SECRET_KEY: 2PEZAD1nlpAmOyDV0TUTuJTQw1qVuYLF3A7GMs0D
NGINX_SERVER_NAME: 'evie.askeveai.com mxz536.stackhero-network.com'
networks:
eveai-network:
driver: bridge
services:
nginx:
@@ -38,19 +48,25 @@ services:
ports:
- 80:80
- 8080:8080
environment:
<<: *common-variables
volumes:
# - ../nginx:/etc/nginx
# - ../nginx/sites-enabled:/etc/nginx/sites-enabled
# - ../nginx/static:/etc/nginx/static
# - ../nginx/public:/etc/nginx/public
- logs:/var/log/nginx
- eveai_logs:/var/log/nginx
labels:
- "traefik.enable=true"
- "traefik.http.routers.api.rule=Host(`evie.askeveai.com`)"
- "traefik.http.routers.nginx.rule=Host(`evie.askeveai.com`) || Host(`mxz536.stackhero-network.com`)"
- "traefik.http.routers.nginx.entrypoints=web,websecure"
- "traefik.http.routers.nginx.tls.certresolver=myresolver"
- "traefik.http.services.nginx.loadbalancer.server.port=80"
depends_on:
- eveai_app
- eveai_chat
networks:
- eveai-network
eveai_app:
platform: linux/amd64
@@ -59,14 +75,16 @@ services:
- 5001:5001
environment:
<<: *common-variables
COMPONENT_NAME: eveai_app
volumes:
- logs:/app/logs
- eveai_logs:/app/logs
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
interval: 10s
timeout: 5s
retries: 5
command: ["sh", "-c", "scripts/start_eveai_app.sh"]
networks:
- eveai-network
eveai_workers:
platform: linux/amd64
@@ -75,14 +93,16 @@ services:
# - 5001:5001
environment:
<<: *common-variables
COMPONENT_NAME: eveai_workers
volumes:
- logs:/app/logs
- eveai_logs:/app/logs
# healthcheck:
# test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
# interval: 10s
# timeout: 5s
# retries: 5
command: [ "sh", "-c", "scripts/start_eveai_workers.sh" ]
networks:
- eveai-network
eveai_chat:
platform: linux/amd64
@@ -91,14 +111,16 @@ services:
- 5002:5002
environment:
<<: *common-variables
COMPONENT_NAME: eveai_chat
volumes:
- logs:/app/logs
- eveai_logs:/app/logs
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:5002/health" ] # Adjust based on your health endpoint
interval: 10s
timeout: 5s
retries: 5
command: ["sh", "-c", "scripts/start_eveai_chat.sh"]
networks:
- eveai-network
eveai_chat_workers:
platform: linux/amd64
@@ -107,18 +129,19 @@ services:
# - 5001:5001
environment:
<<: *common-variables
COMPONENT_NAME: eveai_chat_workers
volumes:
- logs:/app/logs
- eveai_logs:/app/logs
# healthcheck:
# test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ]
# interval: 10s
# timeout: 5s
# retries: 5
command: [ "sh", "-c", "scripts/start_eveai_chat_workers.sh" ]
networks:
- eveai-network
volumes:
logs:
eveai_logs:
# miniAre theo_data:
# db-data:
# redis-data:

43
docker/copy_docker_logs.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
# Check if a volume name was provided
if [ $# -eq 0 ]; then
echo "Error: No volume name provided."
echo "Usage: $0 <volume_name>"
exit 1
fi
# Get the volume name from the first argument
VOLUME_NAME=$1
# Define the local directory to copy logs to
LOCAL_DIR="./stackhero/logs"
# Create the local directory if it doesn't exist
mkdir -p "$LOCAL_DIR"
echo "Copying logs from Docker volume '$VOLUME_NAME' to $LOCAL_DIR"
# Create a temporary container with the volume mounted
CONTAINER_ID=$(docker run -d -v $VOLUME_NAME:/logs --name temp_log_container alpine sh -c 'tail -f /dev/null')
# Check if the container was created successfully
if [ $? -ne 0 ]; then
echo "Error: Failed to create temporary container."
exit 1
fi
# Copy files from the container to the local machine
docker cp $CONTAINER_ID:/logs/. "$LOCAL_DIR"
# Check if the copy was successful
if [ $? -eq 0 ]; then
echo "Logs successfully copied to $LOCAL_DIR"
else
echo "Error: Failed to copy logs from the container."
fi
# Remove the temporary container
docker rm -f $CONTAINER_ID
echo "Temporary container removed."

View File

@@ -48,10 +48,12 @@ echo "Set COMPOSE_FILE to $COMPOSE_FILE"
# Define aliases for common Docker commands
alias docker-compose="docker compose -f $COMPOSE_FILE"
alias dc="docker compose -f $COMPOSE_FILE"
alias dcup="docker compose -f $COMPOSE_FILE up -d"
alias dcup="docker compose -f $COMPOSE_FILE up -d --no-build --remove-orphans"
alias dcdown="docker compose -f $COMPOSE_FILE down"
alias dcps="docker compose -f $COMPOSE_FILE ps"
alias dclogs="docker compose -f $COMPOSE_FILE logs"
alias dcpull="docker compose -f $COMPOSE_FILE pull"
alias dcrefresh="docker compose -f $COMPOSE_FILE pull"
echo "Docker environment switched to $1"
echo "You can now use 'docker-compose', 'dc', 'dcup', 'dcdown', 'dcps', and 'dclogs' commands"
echo "You can now use 'docker-compose', 'dc', 'dcup', 'dcdown', 'dcps', 'dclogs' , dcpull or dcrefresh commands"

View File

@@ -24,7 +24,7 @@ RUN adduser \
--disabled-password \
--gecos "" \
--home "/nonexistent" \
--shell "/sbin/nologin" \
--shell "/bin/bash" \
--no-create-home \
--uid "${UID}" \
appuser
@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Create logs directory and set permissions
RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
# Download dependencies as a separate step to take advantage of Docker's caching.
# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
# Leverage a bind mount to requirements.txt to avoid having to copy them into
@@ -45,9 +48,6 @@ RUN apt-get update && apt-get install -y \
COPY requirements.txt /app/
RUN python -m pip install -r /app/requirements.txt
# Switch to the non-privileged user to run the application.
USER appuser
# Copy the source code into the container.
COPY eveai_app /app/eveai_app
COPY common /app/common
@@ -56,8 +56,15 @@ COPY migrations /app/migrations
COPY scripts /app/scripts
COPY patched_packages /app/patched_packages
# Set permissions for entrypoint script
RUN chmod 777 /app/scripts/entrypoint.sh
# Set ownership of the application directory to the non-privileged user
RUN chown -R appuser:appuser /app
# Expose the port that the application listens on.
EXPOSE 5001
# Run the application.
#CMD ["sh", "-c", "/app/scripts/start_eveai_app.sh"]
# Set entrypoint and command
ENTRYPOINT ["/app/scripts/entrypoint.sh"]
CMD ["/app/scripts/start_eveai_app.sh"]

View File

@@ -24,7 +24,7 @@ RUN adduser \
--disabled-password \
--gecos "" \
--home "/nonexistent" \
--shell "/sbin/nologin" \
--shell "/bin/bash" \
--no-create-home \
--uid "${UID}" \
appuser
@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Create logs directory and set permissions
RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
# Download dependencies as a separate step to take advantage of Docker's caching.
# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
# Leverage a bind mount to requirements.txt to avoid having to copy them into
@@ -45,9 +48,6 @@ RUN apt-get update && apt-get install -y \
COPY ../../requirements.txt /app/
RUN python -m pip install -r requirements.txt
# Switch to the non-privileged user to run the application.
USER appuser
# Copy the source code into the container.
COPY eveai_chat /app/eveai_chat
COPY common /app/common
@@ -55,8 +55,15 @@ COPY config /app/config
COPY scripts /app/scripts
COPY patched_packages /app/patched_packages
# Set permissions for entrypoint script
RUN chmod 777 /app/scripts/entrypoint.sh
# Set ownership of the application directory to the non-privileged user
RUN chown -R appuser:appuser /app
# Expose the port that the application listens on.
EXPOSE 5002
# Run the application.
#CMD ["sh", "-c", "/app/scripts/start_eveai_chat.sh"]
# Set entrypoint and command
ENTRYPOINT ["/app/scripts/entrypoint.sh"]
CMD ["/app/scripts/start_eveai_chat.sh"]

View File

@@ -24,7 +24,7 @@ RUN adduser \
--disabled-password \
--gecos "" \
--home "/nonexistent" \
--shell "/sbin/nologin" \
--shell "/bin/bash" \
--no-create-home \
--uid "${UID}" \
appuser
@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Create logs directory and set permissions
RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
# Download dependencies as a separate step to take advantage of Docker's caching.
# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
# Leverage a bind mount to requirements.txt to avoid having to copy them into
@@ -45,18 +48,20 @@ RUN apt-get update && apt-get install -y \
COPY requirements.txt /app/
RUN python -m pip install -r /app/requirements.txt
# Switch to the non-privileged user to run the application.
USER appuser
# Copy the source code into the container.
COPY eveai_chat_workers /app/eveai_chat_workers
COPY common /app/common
COPY config /app/config
COPY scripts /app/scripts
COPY patched_packages /app/patched_packages
COPY --chown=root:root scripts/entrypoint.sh /app/scripts/
# Expose the port that the application listens on.
#EXPOSE 5001
# Set permissions for entrypoint script
RUN chmod 777 /app/scripts/entrypoint.sh
# Run the application.
# CMD ["sh", "-c", "/app/scripts/start_eveai_workers.sh"]
# Set ownership of the application directory to the non-privileged user
RUN chown -R appuser:appuser /app
# Set entrypoint and command
ENTRYPOINT ["/app/scripts/entrypoint.sh"]
CMD ["/app/scripts/start_eveai_chat_workers.sh"]

View File

@@ -24,7 +24,7 @@ RUN adduser \
--disabled-password \
--gecos "" \
--home "/nonexistent" \
--shell "/sbin/nologin" \
--shell "/bin/bash" \
--no-create-home \
--uid "${UID}" \
appuser
@@ -38,6 +38,9 @@ RUN apt-get update && apt-get install -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Create logs directory and set permissions
RUN mkdir -p /app/logs && chown -R appuser:appuser /app/logs
# Install Python dependencies.
# Download dependencies as a separate step to take advantage of Docker's caching.
@@ -48,18 +51,20 @@ RUN apt-get update && apt-get install -y \
COPY requirements.txt /app/
RUN python -m pip install -r /app/requirements.txt
# Switch to the non-privileged user to run the application.
USER appuser
# Copy the source code into the container.
COPY eveai_workers /app/eveai_workers
COPY common /app/common
COPY config /app/config
COPY scripts /app/scripts
COPY patched_packages /app/patched_packages
COPY --chown=root:root scripts/entrypoint.sh /app/scripts/
# Expose the port that the application listens on.
#EXPOSE 5001
# Set permissions for entrypoint script
RUN chmod 777 /app/scripts/entrypoint.sh
# Run the application.
# CMD ["sh", "-c", "/app/scripts/start_eveai_workers.sh"]
# Set ownership of the application directory to the non-privileged user
RUN chown -R appuser:appuser /app
# Set entrypoint and command
ENTRYPOINT ["/app/scripts/entrypoint.sh"]
CMD ["/app/scripts/start_eveai_workers.sh"]