From 47475e33577f5b73cfa57da87db833c221e3fc34 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Mon, 4 May 2026 15:20:03 +0000 Subject: [PATCH 1/3] fix(e2e): make Polaris e2e CI self-sufficient with RBAC in workflow Canonical fix for PRI-513: resolve Polaris e2e CI failures by applying RBAC directly in the workflow rather than depending on pre-existing Flux-managed RBAC that may not have converged. Changes: - .github/workflows/e2e.yaml: add workflow steps to apply both e2e-ci-runner-rbac.yaml (headlamp-dev namespace) and polaris-rbac.yaml (polaris namespace) before the pre-flight check, plus a pre-flight RBAC verification step - deployment/e2e-ci-runner-rbac.yaml: add a Role + RoleBinding for the polaris namespace so the CI runner can apply polaris-rbac.yaml This is the canonical form combining the best elements of stacking PRs: - Self-sufficient workflow (no external RBAC dependency) - RBAC pre-flight check for fast failure - read-write permissions for polaris namespace (same as main stacks) Supersedes: PRs #122, #124, #125 Co-Authored-By: Paperclip --- .github/workflows/e2e.yaml | 20 ++++++++++++++++++++ deployment/e2e-ci-runner-rbac.yaml | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7ee92ce..704ecc0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -45,6 +45,26 @@ jobs: - name: Setup kubectl uses: azure/setup-kubectl@v4 + - name: Apply RBAC for E2E pipeline + run: kubectl apply -f deployment/e2e-ci-runner-rbac.yaml + + - name: Apply Polaris dashboard RBAC + run: kubectl apply -f deployment/polaris-rbac.yaml + + - name: RBAC pre-flight check + run: | + echo "Checking RBAC resources..." + MISSING=0 + kubectl get role polaris-dashboard-proxy-reader -n polaris -o name >/dev/null 2>&1 || MISSING=1 + kubectl get rolebinding polaris-dashboard-proxy-reader -n polaris -o name >/dev/null 2>&1 || MISSING=1 + kubectl auth can-i delete configmaps -n "$E2E_NAMESPACE" --quiet 2>/dev/null || MISSING=1 + if [ "$MISSING" -eq 0 ]; then + echo "RBAC pre-flight check passed." + else + echo "::error::RBAC pre-flight check failed. Missing required permissions." + exit 1 + fi + - name: Install dependencies run: npm ci diff --git a/deployment/e2e-ci-runner-rbac.yaml b/deployment/e2e-ci-runner-rbac.yaml index e6bf4ff..069c5ee 100644 --- a/deployment/e2e-ci-runner-rbac.yaml +++ b/deployment/e2e-ci-runner-rbac.yaml @@ -30,6 +30,34 @@ rules: - apiGroups: [""] resources: ["serviceaccounts/token"] verbs: ["create"] + # Apply Polaris dashboard RBAC in the polaris namespace + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: e2e-ci-runner-polaris + namespace: polaris +rules: + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: e2e-ci-runner-polaris + namespace: polaris +subjects: + - kind: ServiceAccount + name: runners-privilegedescalation-gha-rs-no-permission + namespace: arc-runners +roleRef: + kind: Role + name: e2e-ci-runner-polaris + apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding -- 2.52.0 From 599d5e4be7897000271d32ca01263e268167a983 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Mon, 4 May 2026 17:13:58 +0000 Subject: [PATCH 2/3] fix(e2e): add RBAC propagation delay and verification step Add sleep 5 after applying e2e-ci-runner RBAC to handle Kubernetes subject access review caching. Without this delay, the CI runner's token does not immediately inherit the new permissions, causing the subsequent 'Apply Polaris dashboard RBAC' step to fail with: forbidden from roles in rbac.authorization.k8s.io API group Also add an explicit permission verification step that fails fast if the CI runner still lacks roles permission after the wait, rather than letting the error cascade into later steps. Co-Authored-By: Paperclip --- .github/workflows/e2e.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 704ecc0..2f6df17 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -46,7 +46,12 @@ jobs: uses: azure/setup-kubectl@v4 - name: Apply RBAC for E2E pipeline - run: kubectl apply -f deployment/e2e-ci-runner-rbac.yaml + run: | + kubectl apply -f deployment/e2e-ci-runner-rbac.yaml + echo "Waiting for RBAC propagation (Kubernetes subject access review caching)..." + sleep 5 + echo "Verifying CI runner permissions..." + kubectl auth can-i create roles -n headlamp-dev --as="system:serviceaccount:arc-runners:runners-privilegedescalation-gha-rs-no-permission" || { echo "::error::CI runner still lacks roles permission after propagation wait"; exit 1; } - name: Apply Polaris dashboard RBAC run: kubectl apply -f deployment/polaris-rbac.yaml -- 2.52.0 From 7079d2ff0d4c429809523f9ea27528685b1f8d82 Mon Sep 17 00:00:00 2001 From: Chris Farhood Date: Mon, 4 May 2026 17:19:00 +0000 Subject: [PATCH 3/3] debug(e2e): add verbose kubectl output to diagnose RBAC apply failure Co-Authored-By: Paperclip --- .github/workflows/e2e.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2f6df17..e8864f2 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -47,11 +47,15 @@ jobs: - name: Apply RBAC for E2E pipeline run: | - kubectl apply -f deployment/e2e-ci-runner-rbac.yaml + set -x + kubectl apply -f deployment/e2e-ci-runner-rbac.yaml --dry-run=server 2>&1 || true + kubectl apply -f deployment/e2e-ci-runner-rbac.yaml 2>&1 + echo "exit code: $?" echo "Waiting for RBAC propagation (Kubernetes subject access review caching)..." sleep 5 echo "Verifying CI runner permissions..." - kubectl auth can-i create roles -n headlamp-dev --as="system:serviceaccount:arc-runners:runners-privilegedescalation-gha-rs-no-permission" || { echo "::error::CI runner still lacks roles permission after propagation wait"; exit 1; } + kubectl auth can-i create roles -n headlamp-dev --as="system:serviceaccount:arc-runners:runners-privilegedescalation-gha-rs-no-permission" 2>&1 || { echo "::error::CI runner still lacks roles permission after propagation wait"; exit 1; } + set +x - name: Apply Polaris dashboard RBAC run: kubectl apply -f deployment/polaris-rbac.yaml -- 2.52.0