fix(test-env): replace Job with initContainer for runner registration

Problem: Job ran on every ArgoCD sync, creating duplicate runners in Gitea.
Solution:
- initContainer gets token + saves to PVC (/data/.registration-token)
- Runner container registers once, persists .runner file in PVC
- Subsequent restarts skip registration (idempotent)
- PVC runner-data (1Gi) persists registration across pod restarts
- Removed register-job.yaml, moved RBAC to rbac.yaml
- Runner waits for DinD before starting
- Stable runner name: test-env-runner
- Labels: edt + ubuntu-latest
This commit is contained in:
XoR
2026-03-12 13:29:36 +03:00
parent 8e06e8a78d
commit bea103a280
4 changed files with 132 additions and 169 deletions

View File

@@ -6,9 +6,7 @@ metadata:
labels:
app: test-env-runner
spec:
replicas: 0 # Scaled by register-job after token is obtained.
# NOTE: requires Docker-in-Docker (DinD) sidecar to run workflows.
# See TODO below for DinD configuration.
replicas: 1
selector:
matchLabels:
app: test-env-runner
@@ -17,6 +15,59 @@ spec:
labels:
app: test-env-runner
spec:
serviceAccountName: runner-registrar
initContainers:
# Obtain registration token from Gitea API once, write to shared volume.
# Uses the same token if .runner file already exists (idempotent).
- name: register
image: alpine/k8s:1.35.1
command:
- sh
- -c
- |
set -e
# If already registered, skip
if [ -f /data/.runner ]; then
echo "Runner already registered, skipping."
exit 0
fi
# Get Gitea admin credentials
USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d)
PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d)
# Resolve Gitea pod IP (headless service)
GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \
-o jsonpath='{.items[0].status.podIP}')
GITEA_URL="http://${GITEA_POD_IP}:3000"
# Wait for Gitea API
for i in $(seq 1 30); do
if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then
break
fi
echo "Waiting for Gitea API... ($i/30)"
sleep 5
done
# Get registration token
TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \
"$GITEA_URL/api/v1/user/actions/runners/registration-token" \
| sed 's/.*"token":"\([^"]*\)".*/\1/')
if [ -z "$TOKEN" ]; then
echo "ERROR: Failed to get registration token"
exit 1
fi
echo "Got token: ${TOKEN:0:8}..."
# Write token for the runner container
echo "$TOKEN" > /data/.registration-token
echo "$GITEA_URL" > /data/.gitea-url
echo "Token saved to /data/.registration-token"
volumeMounts:
- name: data
mountPath: /data
containers:
# Docker-in-Docker sidecar (required for act_runner to execute workflows)
- name: dind
@@ -38,41 +89,39 @@ spec:
memory: 2Gi
- name: runner
image: gitea/act_runner:0.2.11
command:
- sh
- -c
- |
# Wait for Docker daemon
echo "Waiting for Docker daemon..."
for i in $(seq 1 30); do
if docker info > /dev/null 2>&1; then
echo "Docker daemon is ready"
break
fi
sleep 2
done
# Register if not yet registered
if [ ! -f /data/.runner ] && [ -f /data/.registration-token ]; then
TOKEN=$(cat /data/.registration-token)
GITEA_URL=$(cat /data/.gitea-url)
echo "Registering runner at $GITEA_URL..."
act_runner register --no-interactive \
--instance "$GITEA_URL" \
--token "$TOKEN" \
--name "test-env-runner" \
--labels "edt:docker://benadis/ar-edt-slim:latest,ubuntu-latest:docker://node:20-bullseye"
fi
# Start daemon
exec act_runner daemon
env:
- name: DOCKER_HOST
value: "unix:///var/run/docker.sock"
- name: GITEA_INSTANCE_URL
value: "http://gitea-http.gitea.svc.cluster.local:3000"
- name: GITEA_RUNNER_REGISTRATION_TOKEN
valueFrom:
secretKeyRef:
name: test-env-runner-token
key: token
optional: true
# 1C server connection variables (for workflows)
- name: SRV1C_HOST
value: "onec-server.test-env.svc.cluster.local"
- name: SRV1C_PORT
value: "1540"
- name: RAC_HOST
value: "onec-server.test-env.svc.cluster.local"
- name: RAC_PORT
value: "1545"
- name: STORAGE_HOST
value: "onec-server.test-env.svc.cluster.local"
- name: STORAGE_PORT
value: "1542"
- name: PG_HOST
value: "postgres.test-env.svc.cluster.local"
- name: PG_PORT
value: "5432"
- name: PG_USER
value: "usr1cv8"
- name: PG_PASSWORD
valueFrom:
secretKeyRef:
name: test-env-secrets
key: pg-password
volumeMounts:
- name: docker-socket
mountPath: /var/run
@@ -95,4 +144,18 @@ spec:
configMap:
name: test-env-runner-config
- name: data
emptyDir: {}
persistentVolumeClaim:
claimName: runner-data
---
# PVC for runner data — persists .runner registration across pod restarts
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: runner-data
namespace: test-env
spec:
storageClassName: local-path
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 1Gi

View File

@@ -0,0 +1,34 @@
# RBAC for runner registration initContainer
# Allows reading gitea-admin secret and listing pods in gitea namespace
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: runner-registrar
namespace: test-env
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: test-env-gitea-reader
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["gitea-admin"]
verbs: ["get"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: test-env-gitea-reader
subjects:
- kind: ServiceAccount
name: runner-registrar
namespace: test-env
roleRef:
kind: ClusterRole
name: test-env-gitea-reader
apiGroup: rbac.authorization.k8s.io

View File

@@ -1,134 +0,0 @@
# Job: obtains Gitea runner registration token via API and creates
# the test-env-runner-token Secret. Run once after Gitea is available.
#
# Prerequisites: gitea-admin Secret in gitea namespace (created by deploy-k3s)
# The job resolves Gitea pod IP (headless svc) and calls the registration API.
apiVersion: batch/v1
kind: Job
metadata:
name: register-test-env-runner
namespace: test-env
labels:
app: test-env-runner
spec:
backoffLimit: 3
ttlSecondsAfterFinished: 300
template:
spec:
serviceAccountName: runner-registrar
restartPolicy: OnFailure
containers:
- name: register
image: alpine/k8s:1.35.1
command:
- sh
- -c
- |
set -e
echo "=== Obtaining Gitea runner registration token ==="
# Get Gitea admin credentials from gitea namespace
USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d)
PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d)
# Resolve Gitea pod IP (headless service)
GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \
-o jsonpath='{.items[0].status.podIP}')
GITEA_URL="http://${GITEA_POD_IP}:3000"
echo "Gitea URL: $GITEA_URL"
# Wait for Gitea API to be ready
for i in $(seq 1 30); do
if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then
echo "Gitea API is ready"
break
fi
echo "Waiting for Gitea API... ($i/30)"
sleep 5
done
# Get registration token
TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \
"$GITEA_URL/api/v1/user/actions/runners/registration-token" \
| sed 's/.*"token":"\([^"]*\)".*/\1/')
if [ -z "$TOKEN" ]; then
echo "ERROR: Failed to get registration token"
exit 1
fi
echo "Got registration token: ${TOKEN:0:8}..."
# Create/update Secret in test-env namespace
kubectl -n test-env create secret generic test-env-runner-token \
--from-literal=token="$TOKEN" \
--dry-run=client -o yaml | kubectl apply -f -
echo "=== Secret test-env-runner-token created ==="
# Scale runner deployment to 1
kubectl -n test-env scale deployment test-env-runner --replicas=1
echo "=== Runner deployment scaled to 1 ==="
---
# ServiceAccount + RBAC for the registration job
apiVersion: v1
kind: ServiceAccount
metadata:
name: runner-registrar
namespace: test-env
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: runner-registrar
namespace: test-env
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "get", "update", "patch"]
- apiGroups: ["apps"]
resources: ["deployments/scale", "deployments"]
verbs: ["get", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: runner-registrar
namespace: test-env
subjects:
- kind: ServiceAccount
name: runner-registrar
namespace: test-env
roleRef:
kind: Role
name: runner-registrar
apiGroup: rbac.authorization.k8s.io
---
# ClusterRole to read gitea-admin secret from gitea namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: test-env-gitea-reader
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["gitea-admin"]
verbs: ["get"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: test-env-gitea-reader
subjects:
- kind: ServiceAccount
name: runner-registrar
namespace: test-env
roleRef:
kind: ClusterRole
name: test-env-gitea-reader
apiGroup: rbac.authorization.k8s.io

View File

@@ -15,4 +15,4 @@ resources:
# Gitea Actions runner (for apk-ci-ng workflows)
- gitea-runner/deployment.yaml
- gitea-runner/configmap.yaml
- gitea-runner/register-job.yaml
- gitea-runner/rbac.yaml