- Definition of extra eveai_ops service to run (db) jobs

- Definition of manifests for all jobs - Definition of manifests for all eveai services
2025-09-03 15:20:54 +02:00
parent 898bb32318
commit 2a0c92b064
34 changed files with 1345 additions and 26 deletions
--- a/scripts/dbops/00-env-check.sh
+++ b/scripts/dbops/00-env-check.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+# Required env vars
+REQUIRED_VARS=(DB_HOST DB_PORT DB_NAME DB_USER DB_PASS)
+
+MISSING=()
+for v in "${REQUIRED_VARS[@]}"; do
+  if [[ -z "${!v-}" ]]; then MISSING+=("$v"); fi
+done
+
+# Defaults
+export PROJECT_DIR="${PROJECT_DIR:-/app}"
+export FLASK_APP="${FLASK_APP:-${PROJECT_DIR}/scripts/run.py}"
+export COMPONENT_NAME="${COMPONENT_NAME:-eveai_app}"
+export PYTHONPATH="${PYTHONPATH:-${PROJECT_DIR}:${PYTHONPATH-}}"
+
+if ((${#MISSING[@]})); then
+  fail "Missing required env vars: ${MISSING[*]}"
+fi
+
+# Tools check
+need() { command -v "$1" >/dev/null 2>&1 || fail "Required tool not found in PATH: $1"; }
+need psql
+need pg_isready
+need flask
+
+log "Environment OK"
+log "DB_HOST=$DB_HOST DB_PORT=$DB_PORT DB_NAME=$DB_NAME"
+log "PROJECT_DIR=$PROJECT_DIR"
+log "FLASK_APP=$FLASK_APP COMPONENT_NAME=$COMPONENT_NAME"
+
+# Versions (do not leak secrets)
+psql --version || true
+flask --version || true
+
+log "00-env-check completed successfully."
--- a/scripts/dbops/01-wait-for-db.sh
+++ b/scripts/dbops/01-wait-for-db.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+: "${DB_HOST:?DB_HOST required}"
+: "${DB_PORT:?DB_PORT required}"
+: "${DB_NAME:?DB_NAME required}"
+
+INTERVAL=${WAIT_FOR_DB_INTERVAL:-2}
+TIMEOUT=${WAIT_FOR_DB_TIMEOUT:-30}
+
+need() { command -v "$1" >/dev/null 2>&1 || fail "Required tool not found in PATH: $1"; }
+need pg_isready
+
+log "Waiting for database to be ready (timeout=${TIMEOUT}s, interval=${INTERVAL}s)"
+
+end=$((SECONDS + TIMEOUT))
+while (( SECONDS < end )); do
+  if pg_isready -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" >/dev/null 2>&1; then
+    log "Postgres is ready."
+    exit 0
+  fi
+  sleep "$INTERVAL"
+done
+
+fail "Database not ready within ${TIMEOUT}s."
--- a/scripts/dbops/02-db-bootstrap-ext.sh
+++ b/scripts/dbops/02-db-bootstrap-ext.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+: "${DB_HOST:?DB_HOST required}"
+: "${DB_PORT:?DB_PORT required}"
+: "${DB_NAME:?DB_NAME required}"
+: "${DB_USER:?DB_USER required}"
+: "${DB_PASS:?DB_PASS required}"
+
+export PGPASSWORD="$DB_PASS"
+
+log "Ensuring pgvector extension exists in database '$DB_NAME'"
+
+# Verify DB exists and is reachable
+if ! psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -tAc "SELECT 1" >/dev/null 2>&1; then
+  fail "Unable to connect to database '$DB_NAME'. Ensure it exists and credentials are valid."
+fi
+
+psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -v ON_ERROR_STOP=1 -c "CREATE EXTENSION IF NOT EXISTS vector;"
+
+log "pgvector extension ensured (CREATE EXTENSION IF NOT EXISTS vector)."
--- a/scripts/dbops/03-db-migrate-public.sh
+++ b/scripts/dbops/03-db-migrate-public.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+# Env requirements
+REQUIRED_VARS=(DB_HOST DB_PORT DB_NAME DB_USER DB_PASS)
+for v in "${REQUIRED_VARS[@]}"; do : "${!v:?$v required}"; done
+
+export PROJECT_DIR="${PROJECT_DIR:-/app}"
+export FLASK_APP="${FLASK_APP:-${PROJECT_DIR}/scripts/run.py}"
+export COMPONENT_NAME="${COMPONENT_NAME:-eveai_app}"
+export PYTHONPATH="${PYTHONPATH:-${PROJECT_DIR}:${PYTHONPATH-}}"
+
+export PGPASSWORD="$DB_PASS"
+
+log "Applying Alembic migrations to the public schema..."
+flask db upgrade -d "${PROJECT_DIR}/migrations/public"
+log "Finished applying migrations to the public schema."
--- a/scripts/dbops/04-db-migrate-tenant.sh
+++ b/scripts/dbops/04-db-migrate-tenant.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+# Env requirements
+REQUIRED_VARS=(DB_HOST DB_PORT DB_NAME DB_USER DB_PASS)
+for v in "${REQUIRED_VARS[@]}"; do : "${!v:?$v required}"; done
+
+export PROJECT_DIR="${PROJECT_DIR:-/app}"
+export FLASK_APP="${FLASK_APP:-${PROJECT_DIR}/scripts/run.py}"
+export COMPONENT_NAME="${COMPONENT_NAME:-eveai_app}"
+export PYTHONPATH="${PYTHONPATH:-${PROJECT_DIR}:${PYTHONPATH-}}"
+
+export PGPASSWORD="$DB_PASS"
+
+log "Applying Alembic migrations to the tenant schemas (single run handling all tenants)..."
+flask db upgrade -d "${PROJECT_DIR}/migrations/tenant"
+log "Finished applying migrations to the tenant schemas."
--- a/scripts/dbops/05-seed-or-init-data.sh
+++ b/scripts/dbops/05-seed-or-init-data.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+export PROJECT_DIR="${PROJECT_DIR:-/app}"
+SCRIPT_PATH="${PROJECT_DIR}/scripts/initialize_data.py"
+
+[[ -f "$SCRIPT_PATH" ]] || fail "Seed/init script not found: $SCRIPT_PATH"
+
+export FLASK_APP="${FLASK_APP:-${PROJECT_DIR}/scripts/run.py}"
+export COMPONENT_NAME="${COMPONENT_NAME:-eveai_app}"
+export PYTHONPATH="${PYTHONPATH:-${PROJECT_DIR}:${PYTHONPATH-}}"
+
+log "Running initialize_data.py (idempotent one-off per environment)..."
+python "$SCRIPT_PATH"
+log "initialize_data.py completed."
--- a/scripts/dbops/06-verify-minimal.sh
+++ b/scripts/dbops/06-verify-minimal.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+: "${DB_HOST:?DB_HOST required}"
+: "${DB_PORT:?DB_PORT required}"
+: "${DB_NAME:?DB_NAME required}"
+: "${DB_USER:?DB_USER required}"
+: "${DB_PASS:?DB_PASS required}"
+
+export PGPASSWORD="$DB_PASS"
+
+log "Verifying DB connectivity and pgvector extension in '$DB_NAME'"
+
+# Connectivity check
+psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -tAc "SELECT 1" >/dev/null 2>&1 || fail "Cannot connect to database '$DB_NAME'"
+
+# pgvector check
+HAS_VECTOR=$(psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -tAc "SELECT 1 FROM pg_extension WHERE extname='vector'" | tr -d '[:space:]')
+if [[ "$HAS_VECTOR" != "1" ]]; then
+  fail "pgvector extension not found in '$DB_NAME'"
+fi
+
+log "Verification OK: DB reachable and pgvector extension present."
--- a/scripts/dbops/90-initial-cluster-db-create.sh
+++ b/scripts/dbops/90-initial-cluster-db-create.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+fail() { echo "ERROR: $*" >&2; exit 1; }
+
+: "${DB_HOST:?DB_HOST required}"
+: "${DB_PORT:?DB_PORT required}"
+: "${DB_NAME:?DB_NAME required}"
+: "${DB_USER:?DB_USER required}"
+: "${DB_PASS:?DB_PASS required}"
+
+export PGPASSWORD="$DB_PASS"
+
+log "Checking if database '$DB_NAME' exists..."
+EXISTS=$(psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='${DB_NAME}'" | tr -d '[:space:]') || true
+
+if [[ "$EXISTS" == "1" ]]; then
+  log "Database '$DB_NAME' already exists. Nothing to do."
+  exit 0
+fi
+
+log "Creating database '$DB_NAME'..."
+psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d postgres -v ON_ERROR_STOP=1 -c "CREATE DATABASE \"$DB_NAME\";"
+log "Database '$DB_NAME' created successfully."
--- a/scripts/dbops/99-cleanup.sh
+++ b/scripts/dbops/99-cleanup.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] Cleanup complete (no-op)."
--- a/scripts/dbops/README.md
+++ b/scripts/dbops/README.md
@@ -0,0 +1,140 @@
+# DB Operations (Manual)
+
+Scripts to run database initialization and migrations manually during your release process. Default wait timeout is 30s with 2s interval.
+
+Required env (typical):
+- DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASS
+- PROJECT_DIR (defaults /app)
+- FLASK_APP (defaults to ${PROJECT_DIR}/scripts/run.py)
+- COMPONENT_NAME (defaults to eveai_app)
+- PYTHONPATH (defaults to ${PROJECT_DIR})
+
+Order (standard: DB already exists):
+1. 00-env-check.sh
+2. 01-wait-for-db.sh  (WAIT_FOR_DB_TIMEOUT=30 WAIT_FOR_DB_INTERVAL=2)
+3. 02-db-bootstrap-ext.sh  (creates pgvector extension if missing)
+4. 03-db-migrate-public.sh
+5. 04-db-migrate-tenant.sh  (single run for all tenants)
+6. 06-verify-minimal.sh
+
+Optional, one-off for a new environment where DB does not exist yet:
+- 90-initial-cluster-db-create.sh (run before step 3)
+
+Optional seed/init data (one-off per env, idempotent):
+- 05-seed-or-init-data.sh
+
+Notes:
+- Scripts are idempotent where applicable.
+- Do not echo secrets. Password is read from DB_PASS via PGPASSWORD env.
+- Ensure you run these inside the app image/container or any environment with psql, pg_isready, flask and the project code present.
+
+
+## Podman quickstart (manual testing)
+
+Below are two simple ways to run these scripts in a Podman-based setup. Choose the approach that best matches your workflow.
+
+Prerequisites:
+- A reachable PostgreSQL 16 instance and credentials (DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASS)
+- Your application code available inside the container at /app (default PROJECT_DIR)
+- Tools available in the container: psql, pg_isready, flask, python
+  - docker/Dockerfile.base already includes psql; use it to build a tools image if needed
+
+Environment variables (typical):
+- DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASS
+- PROJECT_DIR=/app (default)
+- FLASK_APP=/app/scripts/run.py (default)
+- COMPONENT_NAME=eveai_app (default)
+- PYTHONPATH="/app" (default)
+
+Order to run (standard):
+1) scripts/dbops/00-env-check.sh
+2) scripts/dbops/01-wait-for-db.sh  (WAIT_FOR_DB_TIMEOUT=30 WAIT_FOR_DB_INTERVAL=2)
+3) scripts/dbops/02-db-bootstrap-ext.sh
+4) scripts/dbops/03-db-migrate-public.sh
+5) scripts/dbops/04-db-migrate-tenant.sh
+6) scripts/dbops/06-verify-minimal.sh
+
+Optional (one-time for a new environment where the DB does not yet exist):
+- scripts/dbops/90-initial-cluster-db-create.sh (run before step 3)
+
+Optional seed/init (one-time per environment):
+- scripts/dbops/05-seed-or-init-data.sh
+
+---
+
+Approach A — Exec into an existing app container
+1. Start your stack (e.g., via Podman Compose) so the app container and DB are running.
+2. Find the app container name:
+   podman ps
+3. Exec into the container shell:
+   podman exec -it <app-container-name> bash
+4. Export DB variables inside the container (adjust values):
+   export DB_HOST=127.0.0.1
+   export DB_PORT=5432
+   export DB_NAME=eveai
+   export DB_USER=postgres
+   export DB_PASS=yourpassword
+   # Optional (defaults exist)
+   export PROJECT_DIR=/app
+   export FLASK_APP=/app/scripts/run.py
+   export COMPONENT_NAME=eveai_app
+   export PYTHONPATH="/app"
+5. Run the scripts in order, for example:
+   scripts/dbops/00-env-check.sh && \
+   scripts/dbops/01-wait-for-db.sh && \
+   scripts/dbops/02-db-bootstrap-ext.sh && \
+   scripts/dbops/03-db-migrate-public.sh && \
+   scripts/dbops/04-db-migrate-tenant.sh && \
+   scripts/dbops/06-verify-minimal.sh
+6. Optionally seed/init once per environment:
+   scripts/dbops/05-seed-or-init-data.sh
+
+Notes:
+- If your DB runs in a separate container, DB_HOST should be that container's network hostname (often the service name from compose). If using host networking, 127.0.0.1 may work.
+
+---
+
+Approach B — Use a one-off tools container based on Dockerfile.base
+This runs the scripts without depending on a running app container. It mounts the repo at /app and uses the base image with psql installed.
+
+1. Build the tools image (from repo root):
+   podman build -f docker/Dockerfile.base -t eveai/tools:local .
+
+2. Run a one-off container with your env vars and the repo mounted:
+   podman run --rm -it \
+     --name eveai-dbops \
+     --network host \
+     -e DB_HOST=127.0.0.1 \
+     -e DB_PORT=5432 \
+     -e DB_NAME=eveai \
+     -e DB_USER=postgres \
+     -e DB_PASS=yourpassword \
+     -e PROJECT_DIR=/app \
+     -e FLASK_APP=/app/scripts/run.py \
+     -e COMPONENT_NAME=eveai_app \
+     -e PYTHONPATH="/app" \
+     -e WAIT_FOR_DB_TIMEOUT=30 \
+     -e WAIT_FOR_DB_INTERVAL=2 \
+     -v "$(pwd)":/app \
+     -w /app \
+     eveai/tools:local \
+     bash -lc 'scripts/dbops/00-env-check.sh && scripts/dbops/01-wait-for-db.sh && scripts/dbops/02-db-bootstrap-ext.sh && scripts/dbops/03-db-migrate-public.sh && scripts/dbops/04-db-migrate-tenant.sh && scripts/dbops/06-verify-minimal.sh'
+
+3. Optional seed/init once per environment:
+   podman run --rm -it --network host \
+     -e DB_HOST -e DB_PORT -e DB_NAME -e DB_USER -e DB_PASS \
+     -e PROJECT_DIR -e FLASK_APP -e COMPONENT_NAME -e PYTHONPATH \
+     -v "$(pwd)":/app -w /app \
+     eveai/tools:local \
+     bash -lc 'scripts/dbops/05-seed-or-init-data.sh'
+
+Notes:
+- --network host simplifies connectivity on Linux. If you cannot use it, use a user-defined Podman network and set DB_HOST to the DB container hostname.
+- Do not echo secrets; scripts read DB_PASS via the PGPASSWORD env variable automatically.
+- If flask is missing at runtime, ensure your tools image has Python deps installed or run inside your actual app image/container that already has them.
+
+Troubleshooting
+- Connection refused: verify DB_HOST/PORT and network mode; try --network host.
+- Authentication failed: check DB_USER/DB_PASS and that the user has rights on DB_NAME.
+- pgvector missing even after step 3: ensure your DB user has rights to CREATE EXTENSION in DB_NAME.
+- Flask command not found: run within the app image or install project dependencies in the tools image before running.