From bea103a280f798940acd30f007250df663225f35 Mon Sep 17 00:00:00 2001 From: XoR Date: Thu, 12 Mar 2026 13:29:36 +0300 Subject: [PATCH] fix(test-env): replace Job with initContainer for runner registration Problem: Job ran on every ArgoCD sync, creating duplicate runners in Gitea. Solution: - initContainer gets token + saves to PVC (/data/.registration-token) - Runner container registers once, persists .runner file in PVC - Subsequent restarts skip registration (idempotent) - PVC runner-data (1Gi) persists registration across pod restarts - Removed register-job.yaml, moved RBAC to rbac.yaml - Runner waits for DinD before starting - Stable runner name: test-env-runner - Labels: edt + ubuntu-latest --- test-env/gitea-runner/deployment.yaml | 131 +++++++++++++++++------ test-env/gitea-runner/rbac.yaml | 34 ++++++ test-env/gitea-runner/register-job.yaml | 134 ------------------------ test-env/kustomization.yaml | 2 +- 4 files changed, 132 insertions(+), 169 deletions(-) create mode 100644 test-env/gitea-runner/rbac.yaml delete mode 100644 test-env/gitea-runner/register-job.yaml diff --git a/test-env/gitea-runner/deployment.yaml b/test-env/gitea-runner/deployment.yaml index 607796b..c341da7 100644 --- a/test-env/gitea-runner/deployment.yaml +++ b/test-env/gitea-runner/deployment.yaml @@ -6,9 +6,7 @@ metadata: labels: app: test-env-runner spec: - replicas: 0 # Scaled by register-job after token is obtained. - # NOTE: requires Docker-in-Docker (DinD) sidecar to run workflows. - # See TODO below for DinD configuration. + replicas: 1 selector: matchLabels: app: test-env-runner @@ -17,6 +15,59 @@ spec: labels: app: test-env-runner spec: + serviceAccountName: runner-registrar + initContainers: + # Obtain registration token from Gitea API once, write to shared volume. + # Uses the same token if .runner file already exists (idempotent). + - name: register + image: alpine/k8s:1.35.1 + command: + - sh + - -c + - | + set -e + # If already registered, skip + if [ -f /data/.runner ]; then + echo "Runner already registered, skipping." + exit 0 + fi + + # Get Gitea admin credentials + USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d) + PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d) + + # Resolve Gitea pod IP (headless service) + GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \ + -o jsonpath='{.items[0].status.podIP}') + GITEA_URL="http://${GITEA_POD_IP}:3000" + + # Wait for Gitea API + for i in $(seq 1 30); do + if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then + break + fi + echo "Waiting for Gitea API... ($i/30)" + sleep 5 + done + + # Get registration token + TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \ + "$GITEA_URL/api/v1/user/actions/runners/registration-token" \ + | sed 's/.*"token":"\([^"]*\)".*/\1/') + + if [ -z "$TOKEN" ]; then + echo "ERROR: Failed to get registration token" + exit 1 + fi + echo "Got token: ${TOKEN:0:8}..." + + # Write token for the runner container + echo "$TOKEN" > /data/.registration-token + echo "$GITEA_URL" > /data/.gitea-url + echo "Token saved to /data/.registration-token" + volumeMounts: + - name: data + mountPath: /data containers: # Docker-in-Docker sidecar (required for act_runner to execute workflows) - name: dind @@ -38,41 +89,39 @@ spec: memory: 2Gi - name: runner image: gitea/act_runner:0.2.11 + command: + - sh + - -c + - | + # Wait for Docker daemon + echo "Waiting for Docker daemon..." + for i in $(seq 1 30); do + if docker info > /dev/null 2>&1; then + echo "Docker daemon is ready" + break + fi + sleep 2 + done + + # Register if not yet registered + if [ ! -f /data/.runner ] && [ -f /data/.registration-token ]; then + TOKEN=$(cat /data/.registration-token) + GITEA_URL=$(cat /data/.gitea-url) + echo "Registering runner at $GITEA_URL..." + act_runner register --no-interactive \ + --instance "$GITEA_URL" \ + --token "$TOKEN" \ + --name "test-env-runner" \ + --labels "edt:docker://benadis/ar-edt-slim:latest,ubuntu-latest:docker://node:20-bullseye" + fi + + # Start daemon + exec act_runner daemon env: - name: DOCKER_HOST value: "unix:///var/run/docker.sock" - name: GITEA_INSTANCE_URL value: "http://gitea-http.gitea.svc.cluster.local:3000" - - name: GITEA_RUNNER_REGISTRATION_TOKEN - valueFrom: - secretKeyRef: - name: test-env-runner-token - key: token - optional: true - # 1C server connection variables (for workflows) - - name: SRV1C_HOST - value: "onec-server.test-env.svc.cluster.local" - - name: SRV1C_PORT - value: "1540" - - name: RAC_HOST - value: "onec-server.test-env.svc.cluster.local" - - name: RAC_PORT - value: "1545" - - name: STORAGE_HOST - value: "onec-server.test-env.svc.cluster.local" - - name: STORAGE_PORT - value: "1542" - - name: PG_HOST - value: "postgres.test-env.svc.cluster.local" - - name: PG_PORT - value: "5432" - - name: PG_USER - value: "usr1cv8" - - name: PG_PASSWORD - valueFrom: - secretKeyRef: - name: test-env-secrets - key: pg-password volumeMounts: - name: docker-socket mountPath: /var/run @@ -95,4 +144,18 @@ spec: configMap: name: test-env-runner-config - name: data - emptyDir: {} + persistentVolumeClaim: + claimName: runner-data +--- +# PVC for runner data — persists .runner registration across pod restarts +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: runner-data + namespace: test-env +spec: + storageClassName: local-path + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/test-env/gitea-runner/rbac.yaml b/test-env/gitea-runner/rbac.yaml new file mode 100644 index 0000000..1e9dbd1 --- /dev/null +++ b/test-env/gitea-runner/rbac.yaml @@ -0,0 +1,34 @@ +# RBAC for runner registration initContainer +# Allows reading gitea-admin secret and listing pods in gitea namespace +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: runner-registrar + namespace: test-env +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: test-env-gitea-reader +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["gitea-admin"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: test-env-gitea-reader +subjects: + - kind: ServiceAccount + name: runner-registrar + namespace: test-env +roleRef: + kind: ClusterRole + name: test-env-gitea-reader + apiGroup: rbac.authorization.k8s.io diff --git a/test-env/gitea-runner/register-job.yaml b/test-env/gitea-runner/register-job.yaml deleted file mode 100644 index 069ba85..0000000 --- a/test-env/gitea-runner/register-job.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# Job: obtains Gitea runner registration token via API and creates -# the test-env-runner-token Secret. Run once after Gitea is available. -# -# Prerequisites: gitea-admin Secret in gitea namespace (created by deploy-k3s) -# The job resolves Gitea pod IP (headless svc) and calls the registration API. -apiVersion: batch/v1 -kind: Job -metadata: - name: register-test-env-runner - namespace: test-env - labels: - app: test-env-runner -spec: - backoffLimit: 3 - ttlSecondsAfterFinished: 300 - template: - spec: - serviceAccountName: runner-registrar - restartPolicy: OnFailure - containers: - - name: register - image: alpine/k8s:1.35.1 - command: - - sh - - -c - - | - set -e - - echo "=== Obtaining Gitea runner registration token ===" - - # Get Gitea admin credentials from gitea namespace - USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d) - PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d) - - # Resolve Gitea pod IP (headless service) - GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \ - -o jsonpath='{.items[0].status.podIP}') - GITEA_URL="http://${GITEA_POD_IP}:3000" - - echo "Gitea URL: $GITEA_URL" - - # Wait for Gitea API to be ready - for i in $(seq 1 30); do - if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then - echo "Gitea API is ready" - break - fi - echo "Waiting for Gitea API... ($i/30)" - sleep 5 - done - - # Get registration token - TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \ - "$GITEA_URL/api/v1/user/actions/runners/registration-token" \ - | sed 's/.*"token":"\([^"]*\)".*/\1/') - - if [ -z "$TOKEN" ]; then - echo "ERROR: Failed to get registration token" - exit 1 - fi - - echo "Got registration token: ${TOKEN:0:8}..." - - # Create/update Secret in test-env namespace - kubectl -n test-env create secret generic test-env-runner-token \ - --from-literal=token="$TOKEN" \ - --dry-run=client -o yaml | kubectl apply -f - - - echo "=== Secret test-env-runner-token created ===" - - # Scale runner deployment to 1 - kubectl -n test-env scale deployment test-env-runner --replicas=1 - echo "=== Runner deployment scaled to 1 ===" ---- -# ServiceAccount + RBAC for the registration job -apiVersion: v1 -kind: ServiceAccount -metadata: - name: runner-registrar - namespace: test-env ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: runner-registrar - namespace: test-env -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "update", "patch"] - - apiGroups: ["apps"] - resources: ["deployments/scale", "deployments"] - verbs: ["get", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: runner-registrar - namespace: test-env -subjects: - - kind: ServiceAccount - name: runner-registrar - namespace: test-env -roleRef: - kind: Role - name: runner-registrar - apiGroup: rbac.authorization.k8s.io ---- -# ClusterRole to read gitea-admin secret from gitea namespace -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: test-env-gitea-reader -rules: - - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["gitea-admin"] - verbs: ["get"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: test-env-gitea-reader -subjects: - - kind: ServiceAccount - name: runner-registrar - namespace: test-env -roleRef: - kind: ClusterRole - name: test-env-gitea-reader - apiGroup: rbac.authorization.k8s.io diff --git a/test-env/kustomization.yaml b/test-env/kustomization.yaml index 15bcf33..d6fb387 100644 --- a/test-env/kustomization.yaml +++ b/test-env/kustomization.yaml @@ -15,4 +15,4 @@ resources: # Gitea Actions runner (for apk-ci-ng workflows) - gitea-runner/deployment.yaml - gitea-runner/configmap.yaml - - gitea-runner/register-job.yaml + - gitea-runner/rbac.yaml