fix(test-env): replace Job with initContainer for runner registration
Problem: Job ran on every ArgoCD sync, creating duplicate runners in Gitea. Solution: - initContainer gets token + saves to PVC (/data/.registration-token) - Runner container registers once, persists .runner file in PVC - Subsequent restarts skip registration (idempotent) - PVC runner-data (1Gi) persists registration across pod restarts - Removed register-job.yaml, moved RBAC to rbac.yaml - Runner waits for DinD before starting - Stable runner name: test-env-runner - Labels: edt + ubuntu-latest
This commit is contained in:
@@ -6,9 +6,7 @@ metadata:
|
|||||||
labels:
|
labels:
|
||||||
app: test-env-runner
|
app: test-env-runner
|
||||||
spec:
|
spec:
|
||||||
replicas: 0 # Scaled by register-job after token is obtained.
|
replicas: 1
|
||||||
# NOTE: requires Docker-in-Docker (DinD) sidecar to run workflows.
|
|
||||||
# See TODO below for DinD configuration.
|
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: test-env-runner
|
app: test-env-runner
|
||||||
@@ -17,6 +15,59 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: test-env-runner
|
app: test-env-runner
|
||||||
spec:
|
spec:
|
||||||
|
serviceAccountName: runner-registrar
|
||||||
|
initContainers:
|
||||||
|
# Obtain registration token from Gitea API once, write to shared volume.
|
||||||
|
# Uses the same token if .runner file already exists (idempotent).
|
||||||
|
- name: register
|
||||||
|
image: alpine/k8s:1.35.1
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
set -e
|
||||||
|
# If already registered, skip
|
||||||
|
if [ -f /data/.runner ]; then
|
||||||
|
echo "Runner already registered, skipping."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get Gitea admin credentials
|
||||||
|
USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d)
|
||||||
|
PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d)
|
||||||
|
|
||||||
|
# Resolve Gitea pod IP (headless service)
|
||||||
|
GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \
|
||||||
|
-o jsonpath='{.items[0].status.podIP}')
|
||||||
|
GITEA_URL="http://${GITEA_POD_IP}:3000"
|
||||||
|
|
||||||
|
# Wait for Gitea API
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "Waiting for Gitea API... ($i/30)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
# Get registration token
|
||||||
|
TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \
|
||||||
|
"$GITEA_URL/api/v1/user/actions/runners/registration-token" \
|
||||||
|
| sed 's/.*"token":"\([^"]*\)".*/\1/')
|
||||||
|
|
||||||
|
if [ -z "$TOKEN" ]; then
|
||||||
|
echo "ERROR: Failed to get registration token"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Got token: ${TOKEN:0:8}..."
|
||||||
|
|
||||||
|
# Write token for the runner container
|
||||||
|
echo "$TOKEN" > /data/.registration-token
|
||||||
|
echo "$GITEA_URL" > /data/.gitea-url
|
||||||
|
echo "Token saved to /data/.registration-token"
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
containers:
|
containers:
|
||||||
# Docker-in-Docker sidecar (required for act_runner to execute workflows)
|
# Docker-in-Docker sidecar (required for act_runner to execute workflows)
|
||||||
- name: dind
|
- name: dind
|
||||||
@@ -38,41 +89,39 @@ spec:
|
|||||||
memory: 2Gi
|
memory: 2Gi
|
||||||
- name: runner
|
- name: runner
|
||||||
image: gitea/act_runner:0.2.11
|
image: gitea/act_runner:0.2.11
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
# Wait for Docker daemon
|
||||||
|
echo "Waiting for Docker daemon..."
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
if docker info > /dev/null 2>&1; then
|
||||||
|
echo "Docker daemon is ready"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
# Register if not yet registered
|
||||||
|
if [ ! -f /data/.runner ] && [ -f /data/.registration-token ]; then
|
||||||
|
TOKEN=$(cat /data/.registration-token)
|
||||||
|
GITEA_URL=$(cat /data/.gitea-url)
|
||||||
|
echo "Registering runner at $GITEA_URL..."
|
||||||
|
act_runner register --no-interactive \
|
||||||
|
--instance "$GITEA_URL" \
|
||||||
|
--token "$TOKEN" \
|
||||||
|
--name "test-env-runner" \
|
||||||
|
--labels "edt:docker://benadis/ar-edt-slim:latest,ubuntu-latest:docker://node:20-bullseye"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Start daemon
|
||||||
|
exec act_runner daemon
|
||||||
env:
|
env:
|
||||||
- name: DOCKER_HOST
|
- name: DOCKER_HOST
|
||||||
value: "unix:///var/run/docker.sock"
|
value: "unix:///var/run/docker.sock"
|
||||||
- name: GITEA_INSTANCE_URL
|
- name: GITEA_INSTANCE_URL
|
||||||
value: "http://gitea-http.gitea.svc.cluster.local:3000"
|
value: "http://gitea-http.gitea.svc.cluster.local:3000"
|
||||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: test-env-runner-token
|
|
||||||
key: token
|
|
||||||
optional: true
|
|
||||||
# 1C server connection variables (for workflows)
|
|
||||||
- name: SRV1C_HOST
|
|
||||||
value: "onec-server.test-env.svc.cluster.local"
|
|
||||||
- name: SRV1C_PORT
|
|
||||||
value: "1540"
|
|
||||||
- name: RAC_HOST
|
|
||||||
value: "onec-server.test-env.svc.cluster.local"
|
|
||||||
- name: RAC_PORT
|
|
||||||
value: "1545"
|
|
||||||
- name: STORAGE_HOST
|
|
||||||
value: "onec-server.test-env.svc.cluster.local"
|
|
||||||
- name: STORAGE_PORT
|
|
||||||
value: "1542"
|
|
||||||
- name: PG_HOST
|
|
||||||
value: "postgres.test-env.svc.cluster.local"
|
|
||||||
- name: PG_PORT
|
|
||||||
value: "5432"
|
|
||||||
- name: PG_USER
|
|
||||||
value: "usr1cv8"
|
|
||||||
- name: PG_PASSWORD
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: test-env-secrets
|
|
||||||
key: pg-password
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: docker-socket
|
- name: docker-socket
|
||||||
mountPath: /var/run
|
mountPath: /var/run
|
||||||
@@ -95,4 +144,18 @@ spec:
|
|||||||
configMap:
|
configMap:
|
||||||
name: test-env-runner-config
|
name: test-env-runner-config
|
||||||
- name: data
|
- name: data
|
||||||
emptyDir: {}
|
persistentVolumeClaim:
|
||||||
|
claimName: runner-data
|
||||||
|
---
|
||||||
|
# PVC for runner data — persists .runner registration across pod restarts
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: runner-data
|
||||||
|
namespace: test-env
|
||||||
|
spec:
|
||||||
|
storageClassName: local-path
|
||||||
|
accessModes: ["ReadWriteOnce"]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
|||||||
34
test-env/gitea-runner/rbac.yaml
Normal file
34
test-env/gitea-runner/rbac.yaml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# RBAC for runner registration initContainer
|
||||||
|
# Allows reading gitea-admin secret and listing pods in gitea namespace
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: runner-registrar
|
||||||
|
namespace: test-env
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: test-env-gitea-reader
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["secrets"]
|
||||||
|
resourceNames: ["gitea-admin"]
|
||||||
|
verbs: ["get"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods"]
|
||||||
|
verbs: ["get", "list"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: test-env-gitea-reader
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: runner-registrar
|
||||||
|
namespace: test-env
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: test-env-gitea-reader
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
# Job: obtains Gitea runner registration token via API and creates
|
|
||||||
# the test-env-runner-token Secret. Run once after Gitea is available.
|
|
||||||
#
|
|
||||||
# Prerequisites: gitea-admin Secret in gitea namespace (created by deploy-k3s)
|
|
||||||
# The job resolves Gitea pod IP (headless svc) and calls the registration API.
|
|
||||||
apiVersion: batch/v1
|
|
||||||
kind: Job
|
|
||||||
metadata:
|
|
||||||
name: register-test-env-runner
|
|
||||||
namespace: test-env
|
|
||||||
labels:
|
|
||||||
app: test-env-runner
|
|
||||||
spec:
|
|
||||||
backoffLimit: 3
|
|
||||||
ttlSecondsAfterFinished: 300
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
serviceAccountName: runner-registrar
|
|
||||||
restartPolicy: OnFailure
|
|
||||||
containers:
|
|
||||||
- name: register
|
|
||||||
image: alpine/k8s:1.35.1
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
set -e
|
|
||||||
|
|
||||||
echo "=== Obtaining Gitea runner registration token ==="
|
|
||||||
|
|
||||||
# Get Gitea admin credentials from gitea namespace
|
|
||||||
USER=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.username}' | base64 -d)
|
|
||||||
PASS=$(kubectl -n gitea get secret gitea-admin -o jsonpath='{.data.password}' | base64 -d)
|
|
||||||
|
|
||||||
# Resolve Gitea pod IP (headless service)
|
|
||||||
GITEA_POD_IP=$(kubectl -n gitea get pod -l app.kubernetes.io/name=gitea \
|
|
||||||
-o jsonpath='{.items[0].status.podIP}')
|
|
||||||
GITEA_URL="http://${GITEA_POD_IP}:3000"
|
|
||||||
|
|
||||||
echo "Gitea URL: $GITEA_URL"
|
|
||||||
|
|
||||||
# Wait for Gitea API to be ready
|
|
||||||
for i in $(seq 1 30); do
|
|
||||||
if curl -sf "$GITEA_URL/api/v1/version" > /dev/null 2>&1; then
|
|
||||||
echo "Gitea API is ready"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
echo "Waiting for Gitea API... ($i/30)"
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
# Get registration token
|
|
||||||
TOKEN=$(curl -sf -X POST -u "$USER:$PASS" \
|
|
||||||
"$GITEA_URL/api/v1/user/actions/runners/registration-token" \
|
|
||||||
| sed 's/.*"token":"\([^"]*\)".*/\1/')
|
|
||||||
|
|
||||||
if [ -z "$TOKEN" ]; then
|
|
||||||
echo "ERROR: Failed to get registration token"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Got registration token: ${TOKEN:0:8}..."
|
|
||||||
|
|
||||||
# Create/update Secret in test-env namespace
|
|
||||||
kubectl -n test-env create secret generic test-env-runner-token \
|
|
||||||
--from-literal=token="$TOKEN" \
|
|
||||||
--dry-run=client -o yaml | kubectl apply -f -
|
|
||||||
|
|
||||||
echo "=== Secret test-env-runner-token created ==="
|
|
||||||
|
|
||||||
# Scale runner deployment to 1
|
|
||||||
kubectl -n test-env scale deployment test-env-runner --replicas=1
|
|
||||||
echo "=== Runner deployment scaled to 1 ==="
|
|
||||||
---
|
|
||||||
# ServiceAccount + RBAC for the registration job
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: runner-registrar
|
|
||||||
namespace: test-env
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: Role
|
|
||||||
metadata:
|
|
||||||
name: runner-registrar
|
|
||||||
namespace: test-env
|
|
||||||
rules:
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources: ["secrets"]
|
|
||||||
verbs: ["create", "get", "update", "patch"]
|
|
||||||
- apiGroups: ["apps"]
|
|
||||||
resources: ["deployments/scale", "deployments"]
|
|
||||||
verbs: ["get", "update", "patch"]
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: RoleBinding
|
|
||||||
metadata:
|
|
||||||
name: runner-registrar
|
|
||||||
namespace: test-env
|
|
||||||
subjects:
|
|
||||||
- kind: ServiceAccount
|
|
||||||
name: runner-registrar
|
|
||||||
namespace: test-env
|
|
||||||
roleRef:
|
|
||||||
kind: Role
|
|
||||||
name: runner-registrar
|
|
||||||
apiGroup: rbac.authorization.k8s.io
|
|
||||||
---
|
|
||||||
# ClusterRole to read gitea-admin secret from gitea namespace
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: ClusterRole
|
|
||||||
metadata:
|
|
||||||
name: test-env-gitea-reader
|
|
||||||
rules:
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources: ["secrets"]
|
|
||||||
resourceNames: ["gitea-admin"]
|
|
||||||
verbs: ["get"]
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources: ["pods"]
|
|
||||||
verbs: ["get", "list"]
|
|
||||||
---
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
kind: ClusterRoleBinding
|
|
||||||
metadata:
|
|
||||||
name: test-env-gitea-reader
|
|
||||||
subjects:
|
|
||||||
- kind: ServiceAccount
|
|
||||||
name: runner-registrar
|
|
||||||
namespace: test-env
|
|
||||||
roleRef:
|
|
||||||
kind: ClusterRole
|
|
||||||
name: test-env-gitea-reader
|
|
||||||
apiGroup: rbac.authorization.k8s.io
|
|
||||||
@@ -15,4 +15,4 @@ resources:
|
|||||||
# Gitea Actions runner (for apk-ci-ng workflows)
|
# Gitea Actions runner (for apk-ci-ng workflows)
|
||||||
- gitea-runner/deployment.yaml
|
- gitea-runner/deployment.yaml
|
||||||
- gitea-runner/configmap.yaml
|
- gitea-runner/configmap.yaml
|
||||||
- gitea-runner/register-job.yaml
|
- gitea-runner/rbac.yaml
|
||||||
|
|||||||
Reference in New Issue
Block a user