fix(test-env): replace Job with initContainer for runner registration
Problem: Job ran on every ArgoCD sync, creating duplicate runners in Gitea. Solution: - initContainer gets token + saves to PVC (/data/.registration-token) - Runner container registers once, persists .runner file in PVC - Subsequent restarts skip registration (idempotent) - PVC runner-data (1Gi) persists registration across pod restarts - Removed register-job.yaml, moved RBAC to rbac.yaml - Runner waits for DinD before starting - Stable runner name: test-env-runner - Labels: edt + ubuntu-latest
This commit is contained in:
@@ -6,9 +6,7 @@ metadata:
|
||||
labels:
|
||||
app: test-env-runner
|
||||
spec:
|
||||
replicas: 0 # Scaled by register-job after token is obtained.
|
||||
# NOTE: requires Docker-in-Docker (DinD) sidecar to run workflows.
|
||||
# See TODO below for DinD configuration.
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: test-env-runner
|
||||
@@ -17,6 +15,59 @@ spec:
|
||||
labels:
|
||||
app: test-env-runner
|
||||
spec:
|
||||
serviceAccountName: runner-registrar
|
||||
initContainers:
|
||||
# Obtain registration token from Gitea API once, write to shared volume.
|
||||
# Uses the same token if .runner file already exists (idempotent).
|
||||
- name: register
|
||||
image: alpine/k8s:1.35.1
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
# If already registered, skip
|
||||
if [ -f /data/.runner ]; then
|
||||
echo "Runner already registered, skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Get Gitea admin credentials
|
||||
USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d)
|
||||
PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d)
|
||||
|
||||
# Resolve Gitea pod IP (headless service)
|
||||
GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \
|
||||
-o jsonpath='{.items[0].status.podIP}')
|
||||
GITEA_URL="http://${GITEA_POD_IP}:3000"
|
||||
|
||||
# Wait for Gitea API
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
echo "Waiting for Gitea API... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Get registration token
|
||||
TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \
|
||||
"$GITEA_URL/api/v1/user/actions/runners/registration-token" \
|
||||
| sed 's/.*"token":"\([^"]*\)".*/\1/')
|
||||
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "ERROR: Failed to get registration token"
|
||||
exit 1
|
||||
fi
|
||||
echo "Got token: ${TOKEN:0:8}..."
|
||||
|
||||
# Write token for the runner container
|
||||
echo "$TOKEN" > /data/.registration-token
|
||||
echo "$GITEA_URL" > /data/.gitea-url
|
||||
echo "Token saved to /data/.registration-token"
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
containers:
|
||||
# Docker-in-Docker sidecar (required for act_runner to execute workflows)
|
||||
- name: dind
|
||||
@@ -38,41 +89,39 @@ spec:
|
||||
memory: 2Gi
|
||||
- name: runner
|
||||
image: gitea/act_runner:0.2.11
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
# Wait for Docker daemon
|
||||
echo "Waiting for Docker daemon..."
|
||||
for i in $(seq 1 30); do
|
||||
if docker info > /dev/null 2>&1; then
|
||||
echo "Docker daemon is ready"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Register if not yet registered
|
||||
if [ ! -f /data/.runner ] && [ -f /data/.registration-token ]; then
|
||||
TOKEN=$(cat /data/.registration-token)
|
||||
GITEA_URL=$(cat /data/.gitea-url)
|
||||
echo "Registering runner at $GITEA_URL..."
|
||||
act_runner register --no-interactive \
|
||||
--instance "$GITEA_URL" \
|
||||
--token "$TOKEN" \
|
||||
--name "test-env-runner" \
|
||||
--labels "edt:docker://benadis/ar-edt-slim:latest,ubuntu-latest:docker://node:20-bullseye"
|
||||
fi
|
||||
|
||||
# Start daemon
|
||||
exec act_runner daemon
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: "unix:///var/run/docker.sock"
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: "http://gitea-http.gitea.svc.cluster.local:3000"
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: test-env-runner-token
|
||||
key: token
|
||||
optional: true
|
||||
# 1C server connection variables (for workflows)
|
||||
- name: SRV1C_HOST
|
||||
value: "onec-server.test-env.svc.cluster.local"
|
||||
- name: SRV1C_PORT
|
||||
value: "1540"
|
||||
- name: RAC_HOST
|
||||
value: "onec-server.test-env.svc.cluster.local"
|
||||
- name: RAC_PORT
|
||||
value: "1545"
|
||||
- name: STORAGE_HOST
|
||||
value: "onec-server.test-env.svc.cluster.local"
|
||||
- name: STORAGE_PORT
|
||||
value: "1542"
|
||||
- name: PG_HOST
|
||||
value: "postgres.test-env.svc.cluster.local"
|
||||
- name: PG_PORT
|
||||
value: "5432"
|
||||
- name: PG_USER
|
||||
value: "usr1cv8"
|
||||
- name: PG_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: test-env-secrets
|
||||
key: pg-password
|
||||
volumeMounts:
|
||||
- name: docker-socket
|
||||
mountPath: /var/run
|
||||
@@ -95,4 +144,18 @@ spec:
|
||||
configMap:
|
||||
name: test-env-runner-config
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
persistentVolumeClaim:
|
||||
claimName: runner-data
|
||||
---
|
||||
# PVC for runner data — persists .runner registration across pod restarts
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: runner-data
|
||||
namespace: test-env
|
||||
spec:
|
||||
storageClassName: local-path
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
|
||||
34
test-env/gitea-runner/rbac.yaml
Normal file
34
test-env/gitea-runner/rbac.yaml
Normal file
@@ -0,0 +1,34 @@
|
||||
# RBAC for runner registration initContainer
|
||||
# Allows reading gitea-admin secret and listing pods in gitea namespace
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: test-env-gitea-reader
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["gitea-admin"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: test-env-gitea-reader
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: test-env-gitea-reader
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -1,134 +0,0 @@
|
||||
# Job: obtains Gitea runner registration token via API and creates
|
||||
# the test-env-runner-token Secret. Run once after Gitea is available.
|
||||
#
|
||||
# Prerequisites: gitea-admin Secret in gitea namespace (created by deploy-k3s)
|
||||
# The job resolves Gitea pod IP (headless svc) and calls the registration API.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: register-test-env-runner
|
||||
namespace: test-env
|
||||
labels:
|
||||
app: test-env-runner
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
ttlSecondsAfterFinished: 300
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: runner-registrar
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: register
|
||||
image: alpine/k8s:1.35.1
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
|
||||
echo "=== Obtaining Gitea runner registration token ==="
|
||||
|
||||
# Get Gitea admin credentials from gitea namespace
|
||||
USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d)
|
||||
PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d)
|
||||
|
||||
# Resolve Gitea pod IP (headless service)
|
||||
GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \
|
||||
-o jsonpath='{.items[0].status.podIP}')
|
||||
GITEA_URL="http://${GITEA_POD_IP}:3000"
|
||||
|
||||
echo "Gitea URL: $GITEA_URL"
|
||||
|
||||
# Wait for Gitea API to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then
|
||||
echo "Gitea API is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for Gitea API... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Get registration token
|
||||
TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \
|
||||
"$GITEA_URL/api/v1/user/actions/runners/registration-token" \
|
||||
| sed 's/.*"token":"\([^"]*\)".*/\1/')
|
||||
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "ERROR: Failed to get registration token"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Got registration token: ${TOKEN:0:8}..."
|
||||
|
||||
# Create/update Secret in test-env namespace
|
||||
kubectl -n test-env create secret generic test-env-runner-token \
|
||||
--from-literal=token="$TOKEN" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
echo "=== Secret test-env-runner-token created ==="
|
||||
|
||||
# Scale runner deployment to 1
|
||||
kubectl -n test-env scale deployment test-env-runner --replicas=1
|
||||
echo "=== Runner deployment scaled to 1 ==="
|
||||
---
|
||||
# ServiceAccount + RBAC for the registration job
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "update", "patch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments/scale", "deployments"]
|
||||
verbs: ["get", "update", "patch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: runner-registrar
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
# ClusterRole to read gitea-admin secret from gitea namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: test-env-gitea-reader
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["gitea-admin"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: test-env-gitea-reader
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: runner-registrar
|
||||
namespace: test-env
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: test-env-gitea-reader
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -15,4 +15,4 @@ resources:
|
||||
# Gitea Actions runner (for apk-ci-ng workflows)
|
||||
- gitea-runner/deployment.yaml
|
||||
- gitea-runner/configmap.yaml
|
||||
- gitea-runner/register-job.yaml
|
||||
- gitea-runner/rbac.yaml
|
||||
|
||||
Reference in New Issue
Block a user