- Definition and Improvements to job-system

- Definition of k8s pods for application services

scaleway/manifests/base/applications/kustomization.yaml

@@ -5,4 +5,3 @@ resources:
   - verification/
   - frontend/
   - backend/
-  - ops/jobs/

scaleway/manifests/base/applications/ops/jobs/00-env-check-job.yaml

@@ -18,12 +18,38 @@ spec:
         job-type: env-check
     spec:
       restartPolicy: Never
+      imagePullSecrets:
+      - name: scaleway-registry-cred
       containers:
       - name: dbops
-        image: josakola/eveai_ops:latest
+        image: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops:staging
+        imagePullPolicy: Always
         envFrom:
         - secretRef:
             name: eveai-secrets
+        env:
+        - name: FLASK_APP
+          value: "/app/scripts/run.py"
+        - name: COMPONENT_NAME
+          value: "eveai_ops"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: LOGLEVEL
+          value: "debug"
+        - name: ROLE
+          value: "web"
+        - name: PORT
+          value: "8080"
+        - name: WORKERS
+          value: "1"
+        - name: WORKER_CLASS
+          value: "gevent"
+        - name: WORKER_CONN
+          value: "100"
+        - name: MAX_REQUESTS
+          value: "1000"
+        - name: MAX_REQUESTS_JITTER
+          value: "100"
         command: ["/bin/bash","-lc","/app/scripts/dbops/00-env-check.sh"]
         resources:
           requests:

scaleway/manifests/base/applications/ops/jobs/02-db-bootstrap-ext-job.yaml

@@ -19,12 +19,38 @@ spec:
         job-type: db-bootstrap-ext
     spec:
       restartPolicy: Never
+      imagePullSecrets:
+      - name: scaleway-registry-cred
       containers:
       - name: dbops
-        image: josakola/eveai_ops:latest
+        image: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops:staging
+        imagePullPolicy: Always
         envFrom:
         - secretRef:
             name: eveai-secrets
+        env:
+        - name: FLASK_APP
+          value: "/app/scripts/run.py"
+        - name: COMPONENT_NAME
+          value: "eveai_ops"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: LOGLEVEL
+          value: "debug"
+        - name: ROLE
+          value: "web"
+        - name: PORT
+          value: "8080"
+        - name: WORKERS
+          value: "1"
+        - name: WORKER_CLASS
+          value: "gevent"
+        - name: WORKER_CONN
+          value: "100"
+        - name: MAX_REQUESTS
+          value: "1000"
+        - name: MAX_REQUESTS_JITTER
+          value: "100"
         command: ["/bin/bash","-lc","/app/scripts/dbops/02-db-bootstrap-ext.sh"]
         resources:
           requests:

scaleway/manifests/base/applications/ops/jobs/03-db-migrate-public-job.yaml

@@ -19,12 +19,38 @@ spec:
         job-type: db-migrate-public
     spec:
       restartPolicy: Never
+      imagePullSecrets:
+      - name: scaleway-registry-cred
       containers:
       - name: dbops
-        image: josakola/eveai_ops:latest
+        image: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops:staging
+        imagePullPolicy: Always
         envFrom:
         - secretRef:
             name: eveai-secrets
+        env:
+        - name: FLASK_APP
+          value: "/app/scripts/run.py"
+        - name: COMPONENT_NAME
+          value: "eveai_ops"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: LOGLEVEL
+          value: "debug"
+        - name: ROLE
+          value: "web"
+        - name: PORT
+          value: "8080"
+        - name: WORKERS
+          value: "1"
+        - name: WORKER_CLASS
+          value: "gevent"
+        - name: WORKER_CONN
+          value: "100"
+        - name: MAX_REQUESTS
+          value: "1000"
+        - name: MAX_REQUESTS_JITTER
+          value: "100"
         command: ["/bin/bash","-lc","/app/scripts/dbops/03-db-migrate-public.sh"]
         resources:
           requests:

scaleway/manifests/base/applications/ops/jobs/04-db-migrate-tenant-job.yaml

@@ -19,12 +19,38 @@ spec:
         job-type: db-migrate-tenant
     spec:
       restartPolicy: Never
+      imagePullSecrets:
+      - name: scaleway-registry-cred
       containers:
       - name: dbops
-        image: josakola/eveai_ops:latest
+        image: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops:staging
+        imagePullPolicy: Always
         envFrom:
         - secretRef:
             name: eveai-secrets
+        env:
+        - name: FLASK_APP
+          value: "/app/scripts/run.py"
+        - name: COMPONENT_NAME
+          value: "eveai_ops"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: LOGLEVEL
+          value: "debug"
+        - name: ROLE
+          value: "web"
+        - name: PORT
+          value: "8080"
+        - name: WORKERS
+          value: "1"
+        - name: WORKER_CLASS
+          value: "gevent"
+        - name: WORKER_CONN
+          value: "100"
+        - name: MAX_REQUESTS
+          value: "1000"
+        - name: MAX_REQUESTS_JITTER
+          value: "100"
         command: ["/bin/bash","-lc","/app/scripts/dbops/04-db-migrate-tenant.sh"]
         resources:
           requests:

scaleway/manifests/base/applications/ops/jobs/05-seed-or-init-data-job.yaml

@@ -19,12 +19,38 @@ spec:
         job-type: db-seed-or-init
     spec:
       restartPolicy: Never
+      imagePullSecrets:
+      - name: scaleway-registry-cred
       containers:
       - name: dbops
-        image: josakola/eveai_ops:latest
+        image: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops:staging
+        imagePullPolicy: Always
         envFrom:
         - secretRef:
             name: eveai-secrets
+        env:
+        - name: FLASK_APP
+          value: "/app/scripts/run.py"
+        - name: COMPONENT_NAME
+          value: "eveai_ops"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: LOGLEVEL
+          value: "debug"
+        - name: ROLE
+          value: "web"
+        - name: PORT
+          value: "8080"
+        - name: WORKERS
+          value: "1"
+        - name: WORKER_CLASS
+          value: "gevent"
+        - name: WORKER_CONN
+          value: "100"
+        - name: MAX_REQUESTS
+          value: "1000"
+        - name: MAX_REQUESTS_JITTER
+          value: "100"
         command: ["/bin/bash","-lc","/app/scripts/dbops/05-seed-or-init-data.sh"]
         resources:
           requests:

scaleway/manifests/base/applications/ops/jobs/06-verify-minimal-job.yaml

@@ -19,12 +19,38 @@ spec:
         job-type: db-verify-minimal
     spec:
       restartPolicy: Never
+      imagePullSecrets:
+      - name: scaleway-registry-cred
       containers:
       - name: dbops
-        image: josakola/eveai_ops:latest
+        image: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops:staging
+        imagePullPolicy: Always
         envFrom:
         - secretRef:
             name: eveai-secrets
+        env:
+        - name: FLASK_APP
+          value: "/app/scripts/run.py"
+        - name: COMPONENT_NAME
+          value: "eveai_ops"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: LOGLEVEL
+          value: "debug"
+        - name: ROLE
+          value: "web"
+        - name: PORT
+          value: "8080"
+        - name: WORKERS
+          value: "1"
+        - name: WORKER_CLASS
+          value: "gevent"
+        - name: WORKER_CONN
+          value: "100"
+        - name: MAX_REQUESTS
+          value: "1000"
+        - name: MAX_REQUESTS_JITTER
+          value: "100"
         command: ["/bin/bash","-lc","/app/scripts/dbops/06-verify-minimal.sh"]
         resources:
           requests:

scaleway/manifests/base/secrets/scaleway-registry-secret.yaml

@@ -0,0 +1,35 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: scaleway-registry-secret
+  namespace: eveai-staging
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: scaleway-cluster-secret-store
+    kind: ClusterSecretStore
+  target:
+    name: scaleway-registry-cred
+    creationPolicy: Owner
+    template:
+      type: kubernetes.io/dockerconfigjson
+      data:
+        .dockerconfigjson: |
+          {"auths":{ "{{ .SCW_REGISTRY_URL }}": { 
+            "username":"{{ .SCW_REGISTRY_ACCESS_KEY }}",
+            "password":"{{ .SCW_REGISTRY_SECRET_KEY }}",
+            "auth":"{{ printf "%s:%s" .SCW_REGISTRY_ACCESS_KEY .SCW_REGISTRY_SECRET_KEY | b64enc }}"
+          }}}
+  data:
+  - secretKey: SCW_REGISTRY_URL
+    remoteRef:
+      key: name:eveai-registry
+      property: SCW_REGISTRY_URL
+  - secretKey: SCW_REGISTRY_ACCESS_KEY
+    remoteRef:
+      key: name:eveai-registry
+      property: SCW_REGISTRY_ACCESS_KEY
+  - secretKey: SCW_REGISTRY_SECRET_KEY
+    remoteRef:
+      key: name:eveai-registry
+      property: SCW_REGISTRY_SECRET_KEY

scaleway/manifests/overlays/staging/kustomization.yaml

@@ -1,29 +1,43 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: eveai-staging
 
-# Reference base components
 resources:
-- ../../base/
+  - ../../base/applications/frontend
+  - ../../base/applications/backend
 
-# Staging-specific configuration
-namePrefix: ""
-nameSuffix: ""
-
-commonLabels:
-  environment: staging
-  managed-by: kustomize
-
-# Images (can be overridden for staging-specific versions)
 images:
-- name: nginx
-  newTag: alpine
+  - name: josakola/eveai_ops
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_ops
+    newTag: staging
+  - name: josakola/eveai_app
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_app
+    newTag: staging
+  - name: josakola/eveai_api
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_api
+    newTag: staging
+  - name: josakola/eveai_chat_client
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_chat_client
+    newTag: staging
+  - name: josakola/eveai_workers
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_workers
+    newTag: staging
+  - name: josakola/eveai_chat_workers
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_chat_workers
+    newTag: staging
+  - name: josakola/eveai_entitlements
+    newName: rg.fr-par.scw.cloud/eveai-staging/josakola/eveai_entitlements
+    newTag: staging
 
-# ConfigMap and Secret generators for staging-specific values
-configMapGenerator:
-- name: staging-config
-  literals:
-  - ENVIRONMENT=staging
-  - LOG_LEVEL=INFO
-  - DEBUG=false
-
-# Note: Namespace is handled per resource to avoid conflicts
+patches:
+  - target:
+      kind: Deployment
+      namespace: eveai-staging
+    patch: |-
+      - op: add
+        path: /spec/template/spec/imagePullSecrets
+        value:
+          - name: scaleway-registry-cred
+      - op: add
+        path: /spec/template/spec/containers/0/imagePullPolicy
+        value: Always

scaleway/push_to_scaleway.sh

@@ -198,12 +198,17 @@ for SERVICE in "${SERVICE_ARRAY[@]}"; do
     # Construct image names
     LOCAL_VERSION_IMAGE="$LOCAL_REGISTRY/$ACCOUNT/$SERVICE:$VERSION"
     SCALEWAY_VERSION_IMAGE="$SCALEWAY_REGISTRY/$ACCOUNT/$SERVICE:$VERSION"
+    ENV_TAG="$ENVIRONMENT"
+    SCALEWAY_ENV_IMAGE="$SCALEWAY_REGISTRY/$ACCOUNT/$SERVICE:$ENV_TAG"
     
     echo "   📥 Source: $LOCAL_VERSION_IMAGE"
-    echo "   📤 Target: $SCALEWAY_VERSION_IMAGE"
+    echo "   📤 Target (version): $SCALEWAY_VERSION_IMAGE"
+    echo "   🏷️  Extra tag (environment): $SCALEWAY_ENV_IMAGE"
     
     if [[ "$DRY_RUN" == true ]]; then
-        echo "   🔍 [DRY RUN] Would push $LOCAL_VERSION_IMAGE to $SCALEWAY_VERSION_IMAGE"
+        echo "   🔍 [DRY RUN] Would push $LOCAL_VERSION_IMAGE to:"
+        echo "      - $SCALEWAY_VERSION_IMAGE"
+        echo "      - $SCALEWAY_ENV_IMAGE (environment channel tag)"
         PROCESSED_SERVICES+=("$SERVICE")
         continue
     fi
@@ -225,26 +230,41 @@ for SERVICE in "${SERVICE_ARRAY[@]}"; do
     fi
     
     # Tag for Scaleway registry (direct push with same version tag)
-    echo "   🏷️  Tagging for Scaleway registry..."
+    echo "   🏷️  Tagging for Scaleway registry (version)..."
     if ! podman tag "$LOCAL_VERSION_IMAGE" "$SCALEWAY_VERSION_IMAGE"; then
         echo "   ❌ Failed to tag $LOCAL_VERSION_IMAGE as $SCALEWAY_VERSION_IMAGE"
         FAILED_SERVICES+=("$SERVICE")
         continue
     fi
     
-    # Push to Scaleway registry
-    echo "   📤 Pushing to Scaleway registry..."
+    # Push version tag to Scaleway registry
+    echo "   📤 Pushing version tag to Scaleway registry..."
     if ! podman push "$SCALEWAY_VERSION_IMAGE"; then
         echo "   ❌ Failed to push $SCALEWAY_VERSION_IMAGE"
         FAILED_SERVICES+=("$SERVICE")
         continue
     fi
+
+    # Tag and push environment channel tag
+    echo "   🏷️  Tagging environment channel ($ENV_TAG)..."
+    if ! podman tag "$LOCAL_VERSION_IMAGE" "$SCALEWAY_ENV_IMAGE"; then
+        echo "   ❌ Failed to tag $LOCAL_VERSION_IMAGE as $SCALEWAY_ENV_IMAGE"
+        FAILED_SERVICES+=("$SERVICE")
+        continue
+    fi
+    echo "   📤 Pushing environment tag to Scaleway registry..."
+    if ! podman push "$SCALEWAY_ENV_IMAGE"; then
+        echo "   ❌ Failed to push $SCALEWAY_ENV_IMAGE"
+        FAILED_SERVICES+=("$SERVICE")
+        continue
+    fi
     
-    # Clean up local Scaleway tag
-    echo "   🧹 Cleaning up local Scaleway tag..."
+    # Clean up local Scaleway tags
+    echo "   🧹 Cleaning up local Scaleway tags..."
     podman rmi "$SCALEWAY_VERSION_IMAGE" 2>/dev/null || true
+    podman rmi "$SCALEWAY_ENV_IMAGE" 2>/dev/null || true
     
-    echo "   ✅ Successfully pushed $SERVICE version $VERSION to Scaleway"
+    echo "   ✅ Successfully pushed $SERVICE as $VERSION and :$ENV_TAG to Scaleway"
     PROCESSED_SERVICES+=("$SERVICE")
 done