Compare commits

..

1 Commits

Author SHA1 Message Date
Hugh Hackman a3629127b4 Add workflow to auto-recover stuck action_required runs 2026-03-25 05:17:41 +00:00
14 changed files with 105 additions and 111 deletions
-2
View File
@@ -16,5 +16,3 @@ jobs:
dual-approval:
uses: privilegedescalation/.github/.github/workflows/dual-approval-check.yaml@main
secrets: inherit
with:
pr_number: ${{ github.event.pull_request.number }}
+2 -2
View File
@@ -11,7 +11,7 @@ permissions:
contents: read
# Only one E2E run at a time: the shared E2E_RELEASE (headlamp-e2e) in
# headlamp-dev cannot be shared across concurrent runs.
# privilegedescalation-dev cannot be shared across concurrent runs.
# cancel-in-progress: false (queue, don't cancel) — cancelling in-flight
# runs may skip the if: always() teardown, leaving dangling cluster resources.
concurrency:
@@ -19,7 +19,7 @@ concurrency:
cancel-in-progress: false
env:
E2E_NAMESPACE: headlamp-dev
E2E_NAMESPACE: privilegedescalation-dev
E2E_RELEASE: headlamp-e2e
# Pin to a known-good Headlamp version. Using :latest is risky because
# the tag can change between CI runs, causing flaky failures when a newer
-14
View File
@@ -1,14 +0,0 @@
name: Renovate
on:
schedule:
- cron: '0 3 * * *'
workflow_dispatch:
jobs:
renovate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: renovatebot/github-action@v40.3.0
with:
configurationFile: renovate.json
renovate-json5: true
+64
View File
@@ -0,0 +1,64 @@
name: Workflow Recovery
on:
schedule:
- cron: '*/5 * * * *'
workflow_dispatch:
jobs:
recover-stuck-runs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Generate GitHub App token
id: app-token
if: vars.RELEASE_APP_ID != ''
uses: actions/create-github-app-token@v3
with:
app-id: ${{ vars.RELEASE_APP_ID }}
private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
owner: privilegedescalation
- name: Detect and re-run stuck action_required runs
env:
GH_TOKEN: ${{ steps.app-token.outputs.token || github.token }}
run: |
echo "Checking for action_required runs in privilegedescalation org..."
RUNS=$(curl -sf -H "Authorization: Bearer $GH_TOKEN" \
-H "Accept: application/vnd.github+json" \
"https://api.github.com/orgs/privilegedescalation/actions/runs?status=action_required&per_page=50" \
|| echo '{"workflow_runs": []}')
COUNT=$(echo "$RUNS" | jq '.workflow_runs | length')
echo "Found $COUNT action_required runs"
if [ "$COUNT" = "0" ] || [ "$COUNT" = "null" ]; then
echo "No stuck runs found. Exiting."
exit 0
fi
echo "$RUNS" | jq -r '.workflow_runs[] | @json' | while read -r run; do
RUN_ID=$(echo "$run" | jq -r '.id')
WORKFLOW_NAME=$(echo "$run" | jq -r '.name')
REPO=$(echo "$run" | jq -r '.repository.full_name')
BRANCH=$(echo "$run" | jq -r '.head_branch')
CREATED_AT=$(echo "$run" | jq -r '.created_at')
echo "Found stuck run: $WORKFLOW_NAME (#$RUN_ID) on $REPO branch $BRANCH"
echo "Created at: $CREATED_AT"
echo "Re-running..."
RESP=$(curl -sf -X POST \
-H "Authorization: Bearer $GH_TOKEN" \
-H "Accept: application/vnd.github+json" \
"https://api.github.com/repos/$REPO/actions/runs/$RUN_ID/rerun" \
-w "\n%{http_code}")
HTTP_CODE=$(echo "$RESP" | tail -1)
if [ "$HTTP_CODE" = "201" ] || [ "$HTTP_CODE" = "204" ]; then
echo "Successfully re-ran $WORKFLOW_NAME (#$RUN_ID)"
else
echo "Failed to re-run $WORKFLOW_NAME (#$RUN_ID): $HTTP_CODE"
fi
done
+3 -3
View File
@@ -1,4 +1,4 @@
version: "1.1.0"
version: "1.0.0"
name: headlamp-intel-gpu
displayName: Intel GPU
description: >-
@@ -99,7 +99,7 @@ screenshots:
url: https://raw.githubusercontent.com/privilegedescalation/headlamp-intel-gpu-plugin/main/docs/screenshots/03-metrics.svg
annotations:
headlamp/plugin/archive-url: "https://github.com/privilegedescalation/headlamp-intel-gpu-plugin/releases/download/v1.1.0/intel-gpu-1.1.0.tar.gz"
headlamp/plugin/archive-checksum: sha256:e212381f38c331383604b06f6552997fcba5c8b42a3bd828e3b43ed3e5028448
headlamp/plugin/archive-url: "https://github.com/privilegedescalation/headlamp-intel-gpu-plugin/releases/download/v1.0.0/intel-gpu-1.0.0.tar.gz"
headlamp/plugin/archive-checksum: sha256:93d6c531e7c12440c9625138f0645fc0c3521b574d0089492759699b324943f0
headlamp/plugin/version-compat: ">=0.20.0"
headlamp/plugin/distro-compat: "in-cluster,web,app"
+13 -21
View File
@@ -19,18 +19,16 @@ test.describe('Intel GPU plugin smoke tests', () => {
// Should navigate to the overview route
await expect(page).toHaveURL(/\/intel-gpu$/);
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Overview' })
).toBeVisible();
await expect(page.getByRole('heading', { name: /intel.gpu/i })).toBeVisible();
});
test('overview page renders GPU device list or empty state', async ({ page }) => {
await page.goto('/c/main/intel-gpu');
// Overview heading should be present
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Overview' })
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole('heading', { name: /intel.gpu/i })).toBeVisible({
timeout: 15_000,
});
// Either a populated table/list or an empty-state indicator must be visible
const hasTable = await page.locator('table').first().isVisible().catch(() => false);
@@ -45,9 +43,9 @@ test.describe('Intel GPU plugin smoke tests', () => {
test('device plugins page renders or shows empty state', async ({ page }) => {
await page.goto('/c/main/intel-gpu/device-plugins');
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Device Plugins' })
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole('heading', { name: /device plugin/i })).toBeVisible({
timeout: 15_000,
});
const hasTable = await page.locator('table').first().isVisible().catch(() => false);
const hasEmptyState = await page
@@ -63,24 +61,18 @@ test.describe('Intel GPU plugin smoke tests', () => {
// not after clicking the parent entry from the overview. Test route
// accessibility via direct navigation — each route must render its heading.
await page.goto('/c/main/intel-gpu');
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Overview' })
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole('heading', { name: /intel.gpu/i })).toBeVisible({
timeout: 15_000,
});
await page.goto('/c/main/intel-gpu/nodes');
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Nodes' })
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole('heading', { name: /node/i })).toBeVisible({ timeout: 15_000 });
await page.goto('/c/main/intel-gpu/pods');
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Pods' })
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole('heading', { name: /pod/i })).toBeVisible({ timeout: 15_000 });
await page.goto('/c/main/intel-gpu/metrics');
await expect(
page.locator('main').getByRole('heading', { name: 'Intel GPU — Metrics' })
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole('heading', { name: /metric/i })).toBeVisible({ timeout: 15_000 });
});
test('plugin settings page shows intel-gpu plugin entry', async ({ page }) => {
+5 -5
View File
@@ -1,12 +1,12 @@
{
"name": "intel-gpu",
"version": "1.1.0",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "intel-gpu",
"version": "1.1.0",
"version": "1.0.0",
"license": "Apache-2.0",
"devDependencies": {
"@kinvolk/headlamp-plugin": "^0.13.0",
@@ -11600,9 +11600,9 @@
}
},
"node_modules/lodash": {
"version": "4.18.1",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
"integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
"version": "4.17.23",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz",
"integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==",
"dev": true,
"license": "MIT"
},
+2 -4
View File
@@ -1,6 +1,6 @@
{
"name": "intel-gpu",
"version": "1.1.0",
"version": "1.0.0",
"description": "Headlamp plugin for Intel GPU device plugin visibility and monitoring",
"repository": {
"type": "git",
@@ -44,8 +44,6 @@
},
"overrides": {
"tar": "^7.5.11",
"undici": "^7.24.3",
"lodash": ">=4.18.0",
"elliptic": ">=6.6.1"
"undici": "^7.24.3"
}
}
+4 -9
View File
@@ -5,7 +5,7 @@
# a ConfigMap volume mount. No custom Docker images — the plugin is built
# in CI and injected as a ConfigMap.
#
# E2E resources are deployed to the `headlamp-dev` namespace. Nothing
# E2E resources are deployed to the `privilegedescalation-dev` namespace. Nothing
# persists beyond the test run — teardown cleans up all created resources.
#
# Prerequisites:
@@ -14,7 +14,7 @@
# - RBAC applied: kubectl apply -f deployment/e2e-ci-runner-rbac.yaml
#
# Environment:
# E2E_NAMESPACE — namespace for E2E Headlamp (default: headlamp-dev)
# E2E_NAMESPACE — namespace for E2E Headlamp (default: privilegedescalation-dev)
# E2E_RELEASE — release/resource name prefix (default: headlamp-e2e)
# HEADLAMP_VERSION — Headlamp image tag (default: latest)
set -euo pipefail
@@ -22,7 +22,7 @@ set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
DIST_DIR="$REPO_ROOT/dist"
E2E_NAMESPACE="${E2E_NAMESPACE:-headlamp-dev}"
E2E_NAMESPACE="${E2E_NAMESPACE:-privilegedescalation-dev}"
E2E_RELEASE="${E2E_RELEASE:-headlamp-e2e}"
HEADLAMP_VERSION="${HEADLAMP_VERSION:-latest}"
@@ -59,15 +59,10 @@ kubectl create configmap headlamp-intel-gpu-plugin \
--from-file=package.json="$REPO_ROOT/package.json"
# --- Tear down any existing E2E deployment for a clean start ---
# Deleting the Deployment forces a fresh pod (new ReplicaSet) regardless of
# whether the pod spec changed. The ServiceAccount is also deleted for a clean
# token state. The Service is NOT deleted — leaving it in place avoids an
# Endpoints UID race (FailedToUpdateEndpoint) that causes DNS resolution
# failures. kubectl apply below upserts the Service in-place, and the new
# pod's IP is added to the existing Endpoints automatically.
echo ""
echo "Removing any existing E2E deployment (clean-start)..."
kubectl delete deployment "${E2E_RELEASE}" -n "$E2E_NAMESPACE" --ignore-not-found --wait
kubectl delete service "${E2E_RELEASE}" -n "$E2E_NAMESPACE" --ignore-not-found --wait
kubectl delete serviceaccount "${E2E_RELEASE}" -n "$E2E_NAMESPACE" --ignore-not-found --wait
# --- Deploy Headlamp via kubectl apply ---
+2 -2
View File
@@ -4,13 +4,13 @@
# Tears down the dedicated E2E Headlamp instance deployed by deploy-e2e-headlamp.sh.
#
# Environment:
# E2E_NAMESPACE — namespace to clean up (default: headlamp-dev)
# E2E_NAMESPACE — namespace to clean up (default: privilegedescalation-dev)
# E2E_RELEASE — release/resource name prefix (default: headlamp-e2e)
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
E2E_NAMESPACE="${E2E_NAMESPACE:-headlamp-dev}"
E2E_NAMESPACE="${E2E_NAMESPACE:-privilegedescalation-dev}"
E2E_RELEASE="${E2E_RELEASE:-headlamp-e2e}"
echo "=== E2E Headlamp Teardown ==="
-23
View File
@@ -151,27 +151,4 @@ describe('IntelGpuDataProvider', () => {
expect(callCountAfter).toBeGreaterThan(callCountBefore);
});
});
it('treats a hanging CRD request as unavailable after 2s timeout', async () => {
vi.useFakeTimers();
const nodeWrapper = { jsonData: {} };
vi.mocked(K8s.ResourceClasses.Node.useList).mockReturnValue([[nodeWrapper], null] as any);
vi.mocked(K8s.ResourceClasses.Pod.useList).mockReturnValue([[nodeWrapper], null] as any);
vi.mocked(ApiProxy.request)
.mockReturnValueOnce(new Promise(() => {}))
.mockResolvedValueOnce({ items: [] })
.mockResolvedValueOnce({ items: [] })
.mockResolvedValueOnce({ items: [] });
const { result } = renderHook(() => useIntelGpuContext(), { wrapper: Wrapper });
expect(result.current.loading).toBe(true);
vi.advanceTimersByTime(2000);
await act(async () => {});
expect(result.current.crdAvailable).toBe(false);
expect(result.current.loading).toBe(false);
vi.useRealTimers();
});
});
+3 -18
View File
@@ -69,18 +69,6 @@ export function useIntelGpuContext(): IntelGpuContextValue {
// Helpers
// ---------------------------------------------------------------------------
const DEFAULT_REQUEST_TIMEOUT_MS = 2_000;
/** Wraps a promise with a timeout, rejecting if it doesn't settle within ms. */
function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
return Promise.race([
promise,
new Promise<T>((_, reject) =>
setTimeout(() => reject(new Error(`Request timed out after ${ms}ms`)), ms)
),
]);
}
/** Extract raw Kubernetes JSON from Headlamp KubeObject wrappers. */
const extractJsonData = (items: unknown[]): unknown[] =>
items.map(item =>
@@ -120,11 +108,8 @@ export function IntelGpuDataProvider({ children }: { children: React.ReactNode }
try {
// GpuDevicePlugin CRDs — graceful degradation if CRD not installed
try {
const pluginList = await withTimeout(
ApiProxy.request(
`/apis/${INTEL_DEVICE_PLUGIN_API_GROUP}/${INTEL_DEVICE_PLUGIN_API_VERSION}/gpudeviceplugins`
),
DEFAULT_REQUEST_TIMEOUT_MS
const pluginList = await ApiProxy.request(
`/apis/${INTEL_DEVICE_PLUGIN_API_GROUP}/${INTEL_DEVICE_PLUGIN_API_VERSION}/gpudeviceplugins`
);
if (!cancelled && isKubeList(pluginList)) {
setCrdAvailable(true);
@@ -154,7 +139,7 @@ export function IntelGpuDataProvider({ children }: { children: React.ReactNode }
for (const url of pluginPodSelectors) {
try {
const list = await withTimeout(ApiProxy.request(url), DEFAULT_REQUEST_TIMEOUT_MS);
const list = await ApiProxy.request(url);
if (!cancelled && isKubeList(list)) {
const gpuPluginPods = filterIntelGpuPluginPods(list.items);
foundPluginPods.push(...gpuPluginPods);
+1 -3
View File
@@ -106,13 +106,11 @@ describe('MetricsPage', () => {
vi.clearAllMocks();
});
it('shows loader when ctxLoading=true but heading is visible immediately', () => {
it('shows loader when ctxLoading=true', () => {
vi.mocked(useIntelGpuContext).mockReturnValue(makeContext({ loading: true }));
// fetchGpuMetrics should never be called in loading state
vi.mocked(fetchGpuMetrics).mockResolvedValue(null);
render(<MetricsPage />);
// Heading renders immediately, loader appears below it while waiting for context
expect(screen.getByText('Intel GPU — Metrics')).toBeInTheDocument();
expect(screen.getByTestId('loader')).toHaveTextContent('Loading Intel GPU data...');
});
+6 -5
View File
@@ -230,6 +230,10 @@ export default function MetricsPage() {
};
}, [ctxLoading, fetchSeq]);
if (ctxLoading) {
return <Loader title="Loading Intel GPU data..." />;
}
return (
<>
<div
@@ -243,7 +247,7 @@ export default function MetricsPage() {
<SectionHeader title="Intel GPU — Metrics" />
<button
onClick={() => void doFetch()}
disabled={fetching || ctxLoading}
disabled={fetching}
aria-label="Refresh metrics"
style={{
padding: '6px 16px',
@@ -251,18 +255,15 @@ export default function MetricsPage() {
color: 'var(--mui-palette-primary-main, #0071c5)',
border: '1px solid var(--mui-palette-primary-main, #0071c5)',
borderRadius: '4px',
cursor: fetching || ctxLoading ? 'not-allowed' : 'pointer',
cursor: 'pointer',
fontSize: '13px',
fontWeight: 500,
opacity: fetching || ctxLoading ? 0.6 : 1,
}}
>
{fetching ? 'Refreshing…' : 'Refresh'}
</button>
</div>
{ctxLoading && <Loader title="Loading Intel GPU data..." />}
<MetricRequirements />
{fetching && !metrics && <Loader title="Querying Prometheus for GPU metrics..." />}