refactor: redesign E2E to use custom Docker image instead of PVC/kubectl
Replace the PVC + kubectl-patch approach for E2E plugin deployment with a custom Docker image that has the plugin pre-installed. This eliminates all policy-violating operations: - No PVCs in kube-system - No kubectl exec/cp to Headlamp pods - No deployment patching via kubectl - No temporary pods or ConfigMap-based file transfers The new approach builds a Headlamp image with the plugin baked in (Dockerfile.e2e), deploys it as a dedicated instance in the headlamp-e2e namespace via Helm, and tears it down after tests complete. RBAC is scoped to the headlamp-e2e namespace instead of kube-system. Note: .github/workflows/e2e.yaml still needs updating to use the new scripts — that change is delegated to Hugh (CI/CD owner). Closes: privilegedescalation/headlamp-polaris-plugin#72 Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Gandalf the Greybeard
Executable
+103
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy-e2e-headlamp.sh
|
||||
#
|
||||
# Builds a custom Headlamp image with the polaris plugin pre-installed,
|
||||
# pushes it to ghcr.io, and deploys a dedicated E2E Headlamp instance.
|
||||
#
|
||||
# This replaces the old PVC + kubectl-patch approach. The plugin is part
|
||||
# of the container image — no PVCs, no kubectl exec/cp, no deployment
|
||||
# patching required.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Plugin built (dist/ exists)
|
||||
# - Docker or buildx available
|
||||
# - GHCR_TOKEN set (or GH_TOKEN with packages:write)
|
||||
# - kubectl configured with cluster access
|
||||
# - Helm 3 installed
|
||||
#
|
||||
# Environment:
|
||||
# E2E_NAMESPACE — namespace for E2E Headlamp (default: headlamp-e2e)
|
||||
# E2E_RELEASE — Helm release name (default: headlamp-e2e)
|
||||
# HEADLAMP_VERSION — base Headlamp image tag (default: latest)
|
||||
# IMAGE_TAG — tag for the E2E image (default: git SHA)
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
DIST_DIR="$REPO_ROOT/dist"
|
||||
|
||||
E2E_NAMESPACE="${E2E_NAMESPACE:-headlamp-e2e}"
|
||||
E2E_RELEASE="${E2E_RELEASE:-headlamp-e2e}"
|
||||
HEADLAMP_VERSION="${HEADLAMP_VERSION:-latest}"
|
||||
IMAGE_REPO="ghcr.io/privilegedescalation/headlamp-polaris-e2e"
|
||||
IMAGE_TAG="${IMAGE_TAG:-$(git -C "$REPO_ROOT" rev-parse --short HEAD)}"
|
||||
IMAGE="${IMAGE_REPO}:${IMAGE_TAG}"
|
||||
|
||||
if [ ! -d "$DIST_DIR" ]; then
|
||||
echo "ERROR: dist/ not found. Run 'npm run build' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== E2E Headlamp Deployment ==="
|
||||
echo " Image: $IMAGE"
|
||||
echo " Namespace: $E2E_NAMESPACE"
|
||||
echo " Release: $E2E_RELEASE"
|
||||
|
||||
# --- Build and push the custom image ---
|
||||
echo ""
|
||||
echo "Building E2E Headlamp image..."
|
||||
docker build -f "$REPO_ROOT/Dockerfile.e2e" \
|
||||
--build-arg "HEADLAMP_VERSION=${HEADLAMP_VERSION}" \
|
||||
-t "$IMAGE" \
|
||||
"$REPO_ROOT"
|
||||
|
||||
echo "Pushing image to ghcr.io..."
|
||||
docker push "$IMAGE"
|
||||
|
||||
# --- Deploy with Helm ---
|
||||
echo ""
|
||||
echo "Adding Headlamp Helm repo..."
|
||||
helm repo add headlamp https://headlamp-k8s.github.io/headlamp/ --force-update
|
||||
helm repo update
|
||||
|
||||
echo "Creating namespace ${E2E_NAMESPACE} (if needed)..."
|
||||
kubectl create namespace "$E2E_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
echo "Installing/upgrading Headlamp E2E instance..."
|
||||
helm upgrade --install "$E2E_RELEASE" headlamp/headlamp \
|
||||
-n "$E2E_NAMESPACE" \
|
||||
-f "$REPO_ROOT/deployment/headlamp-e2e-values.yaml" \
|
||||
--set "image.registry=ghcr.io" \
|
||||
--set "image.repository=privilegedescalation/headlamp-polaris-e2e" \
|
||||
--set "image.tag=${IMAGE_TAG}" \
|
||||
--wait \
|
||||
--timeout 120s
|
||||
|
||||
echo "Waiting for rollout..."
|
||||
kubectl rollout status "deployment/${E2E_RELEASE}-headlamp" \
|
||||
-n "$E2E_NAMESPACE" --timeout=120s
|
||||
|
||||
# --- Generate a service URL for tests ---
|
||||
SVC_URL="http://${E2E_RELEASE}-headlamp.${E2E_NAMESPACE}.svc.cluster.local"
|
||||
echo ""
|
||||
echo "E2E Headlamp is ready at: ${SVC_URL}"
|
||||
echo " export HEADLAMP_URL=${SVC_URL}"
|
||||
|
||||
# --- Generate a token for test auth ---
|
||||
echo ""
|
||||
echo "Creating service account token for E2E auth..."
|
||||
kubectl create serviceaccount headlamp-e2e-test \
|
||||
-n "$E2E_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
TOKEN=$(kubectl create token headlamp-e2e-test -n "$E2E_NAMESPACE" --duration=1h 2>/dev/null || echo "")
|
||||
if [ -n "$TOKEN" ]; then
|
||||
echo " export HEADLAMP_TOKEN=<generated>"
|
||||
echo ""
|
||||
echo "HEADLAMP_URL=${SVC_URL}" > "$REPO_ROOT/.env.e2e"
|
||||
echo "HEADLAMP_TOKEN=${TOKEN}" >> "$REPO_ROOT/.env.e2e"
|
||||
echo "Wrote .env.e2e with HEADLAMP_URL and HEADLAMP_TOKEN"
|
||||
else
|
||||
echo " WARNING: Could not generate token. Set HEADLAMP_TOKEN manually or use OIDC."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "E2E deployment complete."
|
||||
@@ -1,135 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy-plugin-via-volume.sh
|
||||
#
|
||||
# Copies the built plugin into the shared PVC so Headlamp picks it up.
|
||||
# Uses a temporary Kubernetes Job to write to the PVC — the CI runner
|
||||
# does NOT need the PVC mounted locally.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/deploy-plugin-via-volume.sh
|
||||
#
|
||||
# Environment:
|
||||
# HEADLAMP_NAMESPACE — namespace where Headlamp runs (default: kube-system)
|
||||
# HEADLAMP_DEPLOY — Headlamp deployment name (default: headlamp)
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
HEADLAMP_NAMESPACE="${HEADLAMP_NAMESPACE:-kube-system}"
|
||||
HEADLAMP_DEPLOY="${HEADLAMP_DEPLOY:-headlamp}"
|
||||
|
||||
# The deployed directory name must match the package.json name and
|
||||
# the registerPluginSettings name. Headlamp identifies plugins by
|
||||
# reading package.json from each subdirectory of the plugins dir.
|
||||
PLUGIN_DIR_NAME="headlamp-polaris"
|
||||
DIST_DIR="$REPO_ROOT/dist"
|
||||
|
||||
if [ ! -d "$DIST_DIR" ]; then
|
||||
echo "ERROR: dist/ not found. Run 'npm run build' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Deploying plugin to shared volume via temporary job..."
|
||||
echo " Source: $DIST_DIR"
|
||||
echo " PVC: headlamp-plugins"
|
||||
echo " Plugin: $PLUGIN_DIR_NAME"
|
||||
|
||||
# Create tarball of plugin dist + package.json
|
||||
TAR_FILE=$(mktemp /tmp/plugin-XXXXXX.tar.gz)
|
||||
tar -czf "$TAR_FILE" -C "$DIST_DIR" . -C "$REPO_ROOT" package.json
|
||||
echo " Tarball: $TAR_FILE ($(du -h "$TAR_FILE" | cut -f1))"
|
||||
|
||||
# Find the node where Headlamp is running — the PVC is ReadWriteOnce so
|
||||
# the deploy job must land on the same node to mount it.
|
||||
HEADLAMP_NODE=$(kubectl get pods -n "$HEADLAMP_NAMESPACE" \
|
||||
-l "app.kubernetes.io/name=headlamp" \
|
||||
-o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null || true)
|
||||
if [ -z "$HEADLAMP_NODE" ]; then
|
||||
HEADLAMP_NODE=$(kubectl get pods -n "$HEADLAMP_NAMESPACE" \
|
||||
-l "app.kubernetes.io/instance=headlamp" \
|
||||
-o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null || true)
|
||||
fi
|
||||
if [ -n "$HEADLAMP_NODE" ]; then
|
||||
echo " Headlamp node: $HEADLAMP_NODE (scheduling deploy job there)"
|
||||
fi
|
||||
|
||||
# Clean up any previous deploy resources
|
||||
kubectl delete pod plugin-deploy -n "$HEADLAMP_NAMESPACE" --ignore-not-found --wait=true 2>/dev/null || true
|
||||
kubectl delete configmap plugin-tarball -n "$HEADLAMP_NAMESPACE" --ignore-not-found 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Store the tarball in a ConfigMap (binary-safe via --from-file)
|
||||
echo "Creating ConfigMap with plugin tarball..."
|
||||
kubectl create configmap plugin-tarball \
|
||||
-n "$HEADLAMP_NAMESPACE" \
|
||||
--from-file=plugin.tar.gz="$TAR_FILE"
|
||||
|
||||
# Build the Pod manifest as a temp file to avoid heredoc YAML escaping issues
|
||||
POD_FILE=$(mktemp /tmp/plugin-deploy-pod-XXXXXX.yaml)
|
||||
|
||||
cat > "$POD_FILE" <<'YAMLDOC'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: plugin-deploy
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: deploy
|
||||
image: busybox:1.36
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "Cleaning up stale plugin directories..."
|
||||
rm -rf /plugins/polaris /plugins/headlamp-polaris
|
||||
echo "Extracting plugin to shared volume..."
|
||||
mkdir -p /plugins/PLUGIN_DIR_PLACEHOLDER
|
||||
tar -xzf /tarball/plugin.tar.gz -C /plugins/PLUGIN_DIR_PLACEHOLDER
|
||||
echo "Files deployed:"
|
||||
ls -la /plugins/PLUGIN_DIR_PLACEHOLDER/
|
||||
volumeMounts:
|
||||
- name: plugins
|
||||
mountPath: /plugins
|
||||
- name: tarball
|
||||
mountPath: /tarball
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: plugins
|
||||
persistentVolumeClaim:
|
||||
claimName: headlamp-plugins
|
||||
- name: tarball
|
||||
configMap:
|
||||
name: plugin-tarball
|
||||
YAMLDOC
|
||||
|
||||
# Substitute plugin dir name
|
||||
sed -i "s/PLUGIN_DIR_PLACEHOLDER/${PLUGIN_DIR_NAME}/g" "$POD_FILE"
|
||||
|
||||
# Add nodeName if we know which node Headlamp is on
|
||||
if [ -n "$HEADLAMP_NODE" ]; then
|
||||
sed -i "/restartPolicy: Never/i\\ nodeName: ${HEADLAMP_NODE}" "$POD_FILE"
|
||||
fi
|
||||
|
||||
echo "Starting deploy pod..."
|
||||
kubectl apply -n "$HEADLAMP_NAMESPACE" -f "$POD_FILE"
|
||||
rm -f "$POD_FILE"
|
||||
|
||||
# Wait for the pod to complete (Succeeded phase)
|
||||
echo "Waiting for deploy pod to complete..."
|
||||
kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/plugin-deploy \
|
||||
-n "$HEADLAMP_NAMESPACE" --timeout=120s
|
||||
|
||||
# Show logs
|
||||
kubectl logs plugin-deploy -n "$HEADLAMP_NAMESPACE" 2>/dev/null || true
|
||||
|
||||
# Clean up
|
||||
kubectl delete pod plugin-deploy -n "$HEADLAMP_NAMESPACE" --ignore-not-found 2>/dev/null || true
|
||||
kubectl delete configmap plugin-tarball -n "$HEADLAMP_NAMESPACE" --ignore-not-found 2>/dev/null || true
|
||||
|
||||
rm -f "$TAR_FILE"
|
||||
|
||||
# Restart Headlamp to pick up the new plugin
|
||||
echo "Restarting Headlamp deployment to load plugin..."
|
||||
kubectl rollout restart "deployment/$HEADLAMP_DEPLOY" -n "$HEADLAMP_NAMESPACE"
|
||||
kubectl rollout status "deployment/$HEADLAMP_DEPLOY" -n "$HEADLAMP_NAMESPACE" --timeout=120s
|
||||
|
||||
echo "Plugin deployed successfully."
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# teardown-e2e-headlamp.sh
|
||||
#
|
||||
# Tears down the dedicated E2E Headlamp instance deployed by deploy-e2e-headlamp.sh.
|
||||
#
|
||||
# Environment:
|
||||
# E2E_NAMESPACE — namespace to clean up (default: headlamp-e2e)
|
||||
# E2E_RELEASE — Helm release to uninstall (default: headlamp-e2e)
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
E2E_NAMESPACE="${E2E_NAMESPACE:-headlamp-e2e}"
|
||||
E2E_RELEASE="${E2E_RELEASE:-headlamp-e2e}"
|
||||
|
||||
echo "=== E2E Headlamp Teardown ==="
|
||||
echo " Namespace: $E2E_NAMESPACE"
|
||||
echo " Release: $E2E_RELEASE"
|
||||
|
||||
echo "Uninstalling Helm release..."
|
||||
helm uninstall "$E2E_RELEASE" -n "$E2E_NAMESPACE" 2>/dev/null || echo "Release not found (already removed?)"
|
||||
|
||||
echo "Deleting namespace..."
|
||||
kubectl delete namespace "$E2E_NAMESPACE" --ignore-not-found --wait=false
|
||||
|
||||
# Clean up local env file
|
||||
rm -f "$REPO_ROOT/.env.e2e"
|
||||
|
||||
echo "Teardown complete."
|
||||
Reference in New Issue
Block a user