diff --git a/.github/workflows/ci-failure-rca.yml b/.github/workflows/ci-failure-rca.yml new file mode 100644 index 000000000000..9a06e7fdd75c --- /dev/null +++ b/.github/workflows/ci-failure-rca.yml @@ -0,0 +1,418 @@ +name: CI Failure RCA + +on: + workflow_dispatch: + inputs: + failure_count: + description: 'Number of recent CI failures to analyze (5-20)' + required: true + type: number + default: 10 + workflow_filter: + description: 'Which CI workflows to scan for failures' + required: true + type: choice + options: + - all + - backend + - frontend + - acceptance + default: all + branch_filter: + description: 'Branch to scan (empty = all branches, "master" = master only)' + required: false + type: string + default: '' + severity: + description: 'Severity level — controls depth of analysis' + required: true + type: choice + options: + - deploy-blocked + - flaky-investigation + - general + default: deploy-blocked + +permissions: + contents: read + actions: read + pull-requests: write + deployments: write + issues: write + +env: + CODER_URL: https://coder.sentry.dev + +jobs: + # ────────────────────────────────────────────────────── + # 1. Scan recent CI runs and collect failure context + # ────────────────────────────────────────────────────── + collect-failure-context: + name: Collect CI failure context + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + workspace_name: ${{ steps.name.outputs.workspace_name }} + failures_json: ${{ steps.collect.outputs.failures_json }} + failure_count: ${{ steps.collect.outputs.failure_count }} + summary: ${{ steps.collect.outputs.summary }} + has_failures: ${{ steps.collect.outputs.has_failures }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Compute workspace name + id: name + run: | + WORKSPACE_NAME="ci-rca-$(date +%Y%m%d-%H%M%S)" + echo "workspace_name=${WORKSPACE_NAME}" >> "$GITHUB_OUTPUT" + + - name: Collect recent CI failures + id: collect + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + INPUT_FAILURE_COUNT: ${{ inputs.failure_count }} + INPUT_WORKFLOW_FILTER: ${{ inputs.workflow_filter }} + INPUT_BRANCH_FILTER: ${{ inputs.branch_filter }} + with: + script: | + const { owner, repo } = context.repo; + const maxFailures = Math.min(Math.max(parseInt(process.env.INPUT_FAILURE_COUNT) || 10, 5), 20); + const workflowFilter = process.env.INPUT_WORKFLOW_FILTER; + const branchFilter = process.env.INPUT_BRANCH_FILTER || ''; + + // Map filter names to workflow file names + const WORKFLOW_FILES = { + all: ['backend.yml', 'frontend.yml', 'acceptance.yml'], + backend: ['backend.yml'], + frontend: ['frontend.yml'], + acceptance: ['acceptance.yml'], + }; + const targetWorkflows = WORKFLOW_FILES[workflowFilter] || WORKFLOW_FILES.all; + + core.info(`Scanning ${targetWorkflows.join(', ')} for up to ${maxFailures} failures`); + if (branchFilter) core.info(`Branch filter: ${branchFilter}`); + + // Collect failed runs across all target workflows + const allFailedRuns = []; + + for (const workflowFile of targetWorkflows) { + const params = { + owner, repo, + workflow_id: workflowFile, + status: 'failure', + per_page: Math.min(maxFailures * 2, 50), // fetch extra to account for filtering + }; + if (branchFilter) params.branch = branchFilter; + + try { + const { data } = await github.rest.actions.listWorkflowRuns(params); + for (const run of data.workflow_runs) { + allFailedRuns.push({ + run_id: run.id, + workflow: workflowFile.replace('.yml', ''), + workflow_name: run.name, + head_sha: run.head_sha, + head_branch: run.head_branch, + html_url: run.html_url, + created_at: run.created_at, + pr_number: run.pull_requests?.[0]?.number || null, + pr_url: run.pull_requests?.[0]?.html_url || null, + }); + } + } catch (e) { + core.warning(`Failed to list runs for ${workflowFile}: ${e.message}`); + } + } + + // Sort by most recent first, dedupe by run_id, take top N + allFailedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + const seen = new Set(); + const uniqueRuns = allFailedRuns.filter(r => { + if (seen.has(r.run_id)) return false; + seen.add(r.run_id); + return true; + }).slice(0, maxFailures); + + if (uniqueRuns.length === 0) { + core.setOutput('has_failures', 'false'); + core.setOutput('failure_count', '0'); + core.setOutput('failures_json', '[]'); + core.setOutput('summary', 'No recent CI failures found matching the filter criteria.'); + core.info('No failures found.'); + return; + } + + core.info(`Found ${uniqueRuns.length} failed runs, collecting job-level details...`); + + // For each failed run, collect job-level failure details + const failures = []; + for (const run of uniqueRuns) { + try { + const jobs = await github.paginate( + github.rest.actions.listJobsForWorkflowRun, + { owner, repo, run_id: run.run_id } + ); + + const failedJobs = jobs + .filter(j => j.conclusion === 'failure') + .map(job => { + const failedSteps = (job.steps || []) + .filter(s => s.conclusion === 'failure') + .map(s => ({ name: s.name, number: s.number })); + return { + job_id: job.id, + job_name: job.name, + job_url: job.html_url, + failed_steps: failedSteps, + }; + }); + + failures.push({ + ...run, + failed_jobs: failedJobs, + failed_job_count: failedJobs.length, + }); + } catch (e) { + core.warning(`Failed to get jobs for run ${run.run_id}: ${e.message}`); + failures.push({ ...run, failed_jobs: [], failed_job_count: 0 }); + } + } + + // Build human-readable summary + const summaryLines = failures.map((f, i) => { + const prRef = f.pr_number ? `PR #${f.pr_number}` : f.head_branch; + const jobNames = f.failed_jobs.map(j => j.job_name).join(', ') || 'unknown'; + const shortSha = f.head_sha.slice(0, 7); + return `${i + 1}. **${f.workflow}** — [${shortSha}](${f.html_url}) on ${prRef} — failed: ${jobNames} (${f.created_at})`; + }); + + // Compute cross-failure stats + const jobNameCounts = {}; + const branchCounts = {}; + const shaCounts = {}; + for (const f of failures) { + branchCounts[f.head_branch] = (branchCounts[f.head_branch] || 0) + 1; + shaCounts[f.head_sha] = (shaCounts[f.head_sha] || 0) + 1; + for (const j of f.failed_jobs) { + jobNameCounts[j.job_name] = (jobNameCounts[j.job_name] || 0) + 1; + } + } + + const topJobs = Object.entries(jobNameCounts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([name, count]) => `- \`${name}\`: ${count} failure(s)`); + + const topBranches = Object.entries(branchCounts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([name, count]) => `- \`${name}\`: ${count} failure(s)`); + + const summary = [ + `## CI Failure Scan Results`, + ``, + `**Filter**: workflow=\`${workflowFilter}\`${branchFilter ? `, branch=\`${branchFilter}\`` : ', branch=all'}`, + `**Failures analyzed**: ${failures.length}`, + `**Time range**: ${failures[failures.length - 1]?.created_at || 'N/A'} → ${failures[0]?.created_at || 'N/A'}`, + ``, + `### Most Frequently Failing Jobs`, + ...topJobs, + ``, + `### Failures by Branch`, + ...topBranches, + ``, + `### Individual Failures`, + ...summaryLines, + ].join('\n'); + + core.setOutput('has_failures', 'true'); + core.setOutput('failure_count', String(failures.length)); + core.setOutput('failures_json', JSON.stringify(failures)); + core.setOutput('summary', summary); + + core.info(`Collected ${failures.length} failures with job details.`); + + - name: Download test artifacts for recent failures + if: steps.collect.outputs.has_failures == 'true' + continue-on-error: true + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + FAILURES_JSON: ${{ steps.collect.outputs.failures_json }} + with: + script: | + const fs = require('fs'); + const { owner, repo } = context.repo; + const failures = JSON.parse(process.env.FAILURES_JSON); + + fs.mkdirSync('/tmp/ci-artifacts', { recursive: true }); + + // Download pytest artifacts for the 5 most recent failures (to avoid rate limits) + for (const failure of failures.slice(0, 5)) { + try { + const { data } = await github.rest.actions.listWorkflowRunArtifacts({ + owner, repo, run_id: failure.run_id, + }); + + const pytestArtifacts = data.artifacts.filter( + a => a.name.startsWith('pytest-results') + ); + + for (const artifact of pytestArtifacts.slice(0, 3)) { + try { + const download = await github.rest.actions.downloadArtifact({ + owner, repo, + artifact_id: artifact.id, + archive_format: 'zip', + }); + const fname = `${failure.run_id}-${artifact.name}.zip`; + fs.writeFileSync(`/tmp/ci-artifacts/${fname}`, Buffer.from(download.data)); + core.info(`Downloaded ${fname}`); + } catch (e) { + core.warning(`Failed to download ${artifact.name}: ${e.message}`); + } + } + } catch (e) { + core.warning(`Failed to list artifacts for run ${failure.run_id}: ${e.message}`); + } + } + + - name: Upload failure context bundle + if: steps.collect.outputs.has_failures == 'true' + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: ci-failure-context + path: /tmp/ci-artifacts/ + retention-days: 3 + if-no-files-found: ignore + + # ────────────────────────────────────────────────────── + # 2. Spin up Coder workspace and run the RCA agent + # ────────────────────────────────────────────────────── + rca-analysis: + name: Run RCA in Coder workspace + needs: collect-failure-context + if: needs.collect-failure-context.outputs.has_failures == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + rca_report: ${{ steps.run-agent.outputs.rca_report }} + root_causes: ${{ steps.run-agent.outputs.root_causes }} + action_items: ${{ steps.run-agent.outputs.action_items }} + overall_confidence: ${{ steps.run-agent.outputs.overall_confidence }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Setup Coder + uses: coder/setup-action@4a607a8113d4e676e2d7c34caa20a814bc88bfda # v1.1.0 + with: + access_url: ${{ env.CODER_URL }} + coder_session_token: ${{ secrets.CODER_SESSION_TOKEN }} + + - name: Create Coder workspace + env: + WORKSPACE_NAME: ${{ needs.collect-failure-context.outputs.workspace_name }} + run: scripts/ci-rca/create-workspace.sh + + - name: Wait for workspace ready + env: + WORKSPACE_NAME: ${{ needs.collect-failure-context.outputs.workspace_name }} + run: scripts/ci-rca/wait-for-workspace.sh + + - name: Run RCA agent + id: run-agent + env: + WORKSPACE_NAME: ${{ needs.collect-failure-context.outputs.workspace_name }} + FAILURES_JSON: ${{ needs.collect-failure-context.outputs.failures_json }} + FAILURE_SUMMARY: ${{ needs.collect-failure-context.outputs.summary }} + FAILURE_COUNT: ${{ needs.collect-failure-context.outputs.failure_count }} + SEVERITY: ${{ inputs.severity }} + WORKFLOW_FILTER: ${{ inputs.workflow_filter }} + BRANCH_FILTER: ${{ inputs.branch_filter }} + run: scripts/ci-rca/run-agent.sh + + - name: Destroy Coder workspace + if: always() + env: + WORKSPACE_NAME: ${{ needs.collect-failure-context.outputs.workspace_name }} + run: scripts/ci-rca/destroy-workspace.sh + + # ────────────────────────────────────────────────────── + # 3. Post results to GitHub + # ────────────────────────────────────────────────────── + report-results: + name: Report RCA results + needs: [collect-failure-context, rca-analysis] + if: always() && needs.collect-failure-context.outputs.has_failures == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Create GitHub issue with RCA report + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + FAILURE_SUMMARY: ${{ needs.collect-failure-context.outputs.summary }} + FAILURE_COUNT: ${{ needs.collect-failure-context.outputs.failure_count }} + RCA_REPORT: ${{ needs.rca-analysis.outputs.rca_report || 'RCA agent did not produce a report. Check the workflow run for details.' }} + ROOT_CAUSES: ${{ needs.rca-analysis.outputs.root_causes || 'Unknown' }} + ACTION_ITEMS: ${{ needs.rca-analysis.outputs.action_items || 'Manual investigation required' }} + CONFIDENCE: ${{ needs.rca-analysis.outputs.overall_confidence || 'low' }} + SEVERITY: ${{ inputs.severity }} + WORKFLOW_FILTER: ${{ inputs.workflow_filter }} + with: + script: | + const { owner, repo } = context.repo; + const severity = process.env.SEVERITY; + const failureCount = process.env.FAILURE_COUNT; + const confidence = process.env.CONFIDENCE; + const rcaRunUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + + const severityLabel = { + 'deploy-blocked': 'Deploy Blocked', + 'flaky-investigation': 'Flaky Investigation', + 'general': 'General CI Analysis', + }[severity] || severity; + + const labels = ['ci-failure', 'ci-rca']; + if (severity === 'deploy-blocked') labels.push('deploy-blocked'); + + const body = [ + `## CI Failure Root Cause Analysis`, + ``, + `| Field | Value |`, + `|-------|-------|`, + `| **Severity** | \`${severity}\` |`, + `| **Confidence** | \`${confidence}\` |`, + `| **Failures Analyzed** | ${failureCount} |`, + `| **Workflow Filter** | \`${process.env.WORKFLOW_FILTER}\` |`, + `| **RCA Run** | [View](${rcaRunUrl}) |`, + ``, + `### Root Causes`, + process.env.ROOT_CAUSES, + ``, + `### Action Items`, + process.env.ACTION_ITEMS, + ``, + `---`, + ``, + `
Full RCA Report`, + ``, + process.env.RCA_REPORT, + ``, + `
`, + ``, + `
Failure Scan Context`, + ``, + process.env.FAILURE_SUMMARY, + ``, + `
`, + ].join('\n'); + + const issue = await github.rest.issues.create({ + owner, repo, + title: `[CI RCA] ${severityLabel}: ${failureCount} failures analyzed (confidence: ${confidence})`, + body, + labels, + }); + + core.info(`Created issue: ${issue.data.html_url}`); diff --git a/scripts/ci-rca/create-workspace.sh b/scripts/ci-rca/create-workspace.sh new file mode 100755 index 000000000000..e92c822e2912 --- /dev/null +++ b/scripts/ci-rca/create-workspace.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Create a Coder workspace for CI failure RCA analysis. +# +# Prerequisites: +# coder CLI must be installed and authenticated (via coder/setup-action) +# +# Required env vars: +# CODER_URL - Coder server URL (set by coder/setup-action) +# WORKSPACE_NAME - Computed workspace name (e.g. ci-rca-20260311-143000) + +CODER_API="${CODER_URL}/api/v2" +TEMPLATE_NAME="${TEMPLATE_NAME:-sentry-devbox}" +MACHINE_TYPE="${MACHINE_TYPE:-e2-standard-8}" + +# Read session token from coder CLI config (set by coder login via setup-action) +CODER_TOKEN=$(coder tokens list -o json 2>/dev/null | jq -r '.[0].id // empty' || true) +if [ -z "$CODER_TOKEN" ]; then + # Fallback: read from config file + CODER_TOKEN=$(cat ~/.config/coderv2/session 2>/dev/null || echo "${CODER_SESSION_TOKEN:-}") +fi +AUTH_HEADER="Coder-Session-Token: ${CODER_TOKEN}" + +# curl wrapper that shows response body on failure +api() { + local response http_code + response=$(curl -sS -w "\n%{http_code}" -H "${AUTH_HEADER}" "$@") + http_code=$(echo "$response" | tail -1) + response=$(echo "$response" | sed '$d') + + if [[ "$http_code" -ge 400 ]]; then + # 410 = already deleted, not a real error for our use case + if [[ "$http_code" -eq 410 ]]; then + echo "" + return 0 + fi + echo "ERROR: HTTP ${http_code}" >&2 + echo "$response" >&2 + return 1 + fi + echo "$response" +} + +# ---------- Look up template ---------- + +echo "Looking up template '${TEMPLATE_NAME}'..." +TEMPLATE_ID=$(api "${CODER_API}/templates" | jq -r ".[] | select(.name == \"${TEMPLATE_NAME}\") | .id") + +if [ -z "$TEMPLATE_ID" ]; then + echo "ERROR: Template '${TEMPLATE_NAME}' not found" >&2 + exit 1 +fi + +ACTIVE_VERSION_ID=$(api "${CODER_API}/templates/${TEMPLATE_ID}" | jq -r '.active_version_id') + +echo "Template ID: ${TEMPLATE_ID}, Active Version: ${ACTIVE_VERSION_ID}" + +# ---------- Build parameter payload ---------- + +BUILD_PARAMS=$(jq -n \ + --arg version_id "$ACTIVE_VERSION_ID" \ + --arg machine_type "$MACHINE_TYPE" \ + '{ + template_version_id: $version_id, + transition: "start", + rich_parameter_values: [ + {name: "machine_type", value: $machine_type} + ] + }') + +# ---------- Check if workspace already exists ---------- + +echo "Checking for existing workspace '${WORKSPACE_NAME}'..." +WORKSPACE_JSON=$(api \ + "${CODER_API}/workspaces?q=owner:me+name:${WORKSPACE_NAME}" | jq -r '.workspaces[0] // empty') + +if [ -n "$WORKSPACE_JSON" ]; then + WORKSPACE_ID=$(echo "$WORKSPACE_JSON" | jq -r '.id') + WORKSPACE_STATUS=$(echo "$WORKSPACE_JSON" | jq -r '.latest_build.status') + echo "Workspace exists: ${WORKSPACE_ID} (status: ${WORKSPACE_STATUS})" + + # Wait for any in-progress build to finish + while [[ "$WORKSPACE_STATUS" == "starting" || "$WORKSPACE_STATUS" == "stopping" || \ + "$WORKSPACE_STATUS" == "canceling" || "$WORKSPACE_STATUS" == "pending" || \ + "$WORKSPACE_STATUS" == "deleting" ]]; do + echo " Build in progress (${WORKSPACE_STATUS}), waiting..." + sleep 10 + if ! WORKSPACE_STATUS=$(api \ + "${CODER_API}/workspaces/${WORKSPACE_ID}" | jq -r '.latest_build.status'); then + WORKSPACE_STATUS="deleted" + fi + done + + if [ "$WORKSPACE_STATUS" = "deleted" ]; then + echo "Workspace was deleted, creating fresh..." + WORKSPACE_JSON="" + elif [ "$WORKSPACE_STATUS" = "running" ]; then + echo "Workspace already running, reusing." + else + echo "Workspace is ${WORKSPACE_STATUS}, triggering start..." + api -X POST \ + -H "Content-Type: application/json" \ + "${CODER_API}/workspaces/${WORKSPACE_ID}/builds" \ + -d "$BUILD_PARAMS" > /dev/null + fi +fi + +if [ -z "$WORKSPACE_JSON" ]; then + echo "Creating workspace '${WORKSPACE_NAME}'..." + WORKSPACE_JSON=$(api -X POST \ + -H "Content-Type: application/json" \ + "${CODER_API}/users/me/workspaces" \ + -d "$(jq -n \ + --arg name "$WORKSPACE_NAME" \ + --argjson params "$BUILD_PARAMS" \ + '$params + {name: $name}')") + WORKSPACE_ID=$(echo "$WORKSPACE_JSON" | jq -r '.id') + echo "Created workspace: ${WORKSPACE_ID}" +fi + +echo "Workspace creation initiated." diff --git a/scripts/ci-rca/destroy-workspace.sh b/scripts/ci-rca/destroy-workspace.sh new file mode 100755 index 000000000000..19430324f6cb --- /dev/null +++ b/scripts/ci-rca/destroy-workspace.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Destroy a Coder workspace used for CI RCA analysis. +# Designed to run in the `if: always()` cleanup step. +# +# Prerequisites: +# coder CLI must be installed and authenticated (via coder/setup-action) +# +# Required env vars: +# WORKSPACE_NAME - Workspace name to destroy + +echo "Looking up workspace '${WORKSPACE_NAME}'..." + +# Check if workspace exists +if ! coder show "${WORKSPACE_NAME}" --output json > /dev/null 2>&1; then + echo "Workspace '${WORKSPACE_NAME}' not found, nothing to destroy." + exit 0 +fi + +WORKSPACE_STATUS=$(coder show "${WORKSPACE_NAME}" --output json | jq -r '.latest_build.status // "unknown"') +echo "Found workspace (status: ${WORKSPACE_STATUS})" + +# Wait for any active build to finish before deleting +while [[ "$WORKSPACE_STATUS" == "starting" || "$WORKSPACE_STATUS" == "stopping" || \ + "$WORKSPACE_STATUS" == "deleting" || "$WORKSPACE_STATUS" == "canceling" ]]; do + echo " Build in progress (${WORKSPACE_STATUS}), waiting..." + sleep 10 + WORKSPACE_STATUS=$(coder show "${WORKSPACE_NAME}" --output json 2>/dev/null \ + | jq -r '.latest_build.status // "deleted"') +done + +if [ "$WORKSPACE_STATUS" = "deleted" ]; then + echo "Workspace already deleted." + exit 0 +fi + +echo "Deleting workspace..." +coder delete "${WORKSPACE_NAME}" --yes +echo "Workspace deletion initiated." diff --git a/scripts/ci-rca/run-agent.sh b/scripts/ci-rca/run-agent.sh new file mode 100755 index 000000000000..67b5f0244af0 --- /dev/null +++ b/scripts/ci-rca/run-agent.sh @@ -0,0 +1,293 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run an AI agent inside a Coder workspace to perform RCA across multiple CI failures. +# +# The agent receives structured context about N recent failures and produces +# a cross-failure analysis identifying patterns, root causes, and action items. +# +# Prerequisites: +# coder CLI must be installed and authenticated (via coder/setup-action) +# +# Required env vars: +# WORKSPACE_NAME - Coder workspace name +# FAILURES_JSON - JSON array of failure objects (each with run_id, failed_jobs, etc.) +# FAILURE_SUMMARY - Human-readable summary with stats +# FAILURE_COUNT - Number of failures being analyzed +# SEVERITY - deploy-blocked | flaky-investigation | general +# WORKFLOW_FILTER - Which workflows were scanned +# BRANCH_FILTER - Branch filter (may be empty) + +echo "Resolving workspace '${WORKSPACE_NAME}'..." +WORKSPACE_ID=$(coder show "${WORKSPACE_NAME}" --output json | jq -r '.id') +echo "Workspace: ${WORKSPACE_ID}" + +# ---------- Severity-specific instructions ---------- + +case "$SEVERITY" in + deploy-blocked) + SEVERITY_INSTRUCTIONS=" +CRITICAL: Deploys are blocked. This is the highest priority. + +Your analysis must answer: +1. Is there a single root cause across multiple failures, or are they independent? +2. For each cluster of failures: is it a code regression, flaky test, or infrastructure issue? +3. If code regression: identify the EXACT commit and PR that introduced it, and who authored it. +4. Recommendation priority: revert (with specific commit/PR) > fix-forward (with patch) > retry (if infra flake). +5. For flaky tests blocking deploys: should they be quarantined immediately? +6. Estimate: if we revert/fix the top issue, how many of the ${FAILURE_COUNT} failures would be resolved? +" + ;; + flaky-investigation) + SEVERITY_INSTRUCTIONS=" +This is a flaky test investigation across ${FAILURE_COUNT} failures. Focus on: + +1. Which tests appear in MULTIPLE failures? These are the flakiest — rank by frequency. +2. For each flaky test cluster, identify the root cause pattern: + - Shared mutable state between tests + - Timing/race conditions (async, threading, timeouts) + - Database state leaks (missing cleanup, ordering) + - External service dependencies (redis, kafka, celery) + - Resource contention (memory, file handles, ports) +3. Are failures correlated with specific times of day, branches, or commit patterns? +4. For each flaky test, suggest a concrete fix (not just 'add retry'). +" + ;; + general) + SEVERITY_INSTRUCTIONS=" +General CI failure analysis across ${FAILURE_COUNT} recent failures. Provide: + +1. Categorize each failure: code-bug, flaky-test, infrastructure, dependency, configuration. +2. Identify any patterns or clusters. +3. Prioritize: which failures are most impactful and most fixable? +4. For the top 3 issues, provide specific fix recommendations. +" + ;; +esac + +# ---------- Build the agent prompt ---------- + +AGENT_PROMPT=$(cat <<'PROMPT_HEADER' +You are a CI failure root cause analysis agent for the Sentry codebase. +You are analyzing MULTIPLE recent CI failures to find patterns, common root causes, +and produce actionable recommendations. + +PROMPT_HEADER +) + +AGENT_PROMPT+=$(cat <` +2. Read the failing test code and the code under test. +3. Check `git log --oneline -20` for recent changes to the relevant files. +4. Look at the diff that may have introduced the failure. +5. Try to reproduce: `.venv/bin/pytest -svv --reuse-db ` +6. Determine: is this a real regression, a flaky test, or an infrastructure issue? + +### Phase 3: Cross-failure analysis +1. Look for common threads across failures: + - Same test files appearing in different runs + - Same source files modified in commits that trigger failures + - Time-based patterns (failures started after a specific commit) +2. Identify the "blast radius" — how many failures would be fixed by addressing each root cause? + +### Phase 4: Produce structured output +Write a JSON file to `/tmp/rca-result.json` with this exact schema: + +```json +{ + "root_causes": [ + { + "id": 1, + "title": "Short title of this root cause", + "description": "Detailed explanation with evidence", + "category": "code-bug|flaky-test|infrastructure|dependency|configuration|unknown", + "confidence": "high|medium|low", + "affected_runs": ["list of run_ids affected by this root cause"], + "affected_tests": ["list of test node IDs"], + "relevant_commits": ["commit SHAs that introduced the issue"], + "relevant_files": ["file paths involved"], + "suggested_fix": "Specific actionable fix with file paths and code changes", + "blast_radius": "N of M failures would be fixed" + } + ], + "action_items": [ + { + "priority": 1, + "action": "What to do (e.g., 'Revert PR #12345' or 'Quarantine test_foo')", + "rationale": "Why this is the right action", + "estimated_impact": "How many failures this would resolve" + } + ], + "overall_confidence": "high|medium|low", + "patterns": { + "most_failing_jobs": {"job_name": "count"}, + "most_failing_tests": {"test_nodeid": "count"}, + "failure_timeline": "Description of when failures started and any inflection points", + "is_master_broken": true/false, + "estimated_flake_rate": "X% of failures appear to be flakes vs real regressions" + }, + "full_analysis": "Detailed multi-paragraph analysis with all evidence, code references, and reasoning" +} +``` + +IMPORTANT: +- Be thorough but efficient. Start with the highest-frequency failures. +- Always ground your analysis in actual code — read the files, don't guess. +- If you can't reproduce locally, note that and analyze from code reading alone. +- Write the JSON result file even if your confidence is low. +- Focus on ACTIONABLE output — every root cause should have a suggested fix. +PROMPT_TASK +) + +# ---------- Execute the agent via Coder CLI ---------- + +echo "Launching RCA agent in workspace (analyzing ${FAILURE_COUNT} failures)..." + +# Write prompt to temp file to avoid shell escaping issues +PROMPT_FILE=$(mktemp) +printf '%s' "$AGENT_PROMPT" > "$PROMPT_FILE" + +# Execute Claude Code inside the Coder workspace +coder ssh "${WORKSPACE_NAME}" -- bash -c " + cat > /tmp/rca-prompt.txt << 'INNER_EOF' +$(cat "$PROMPT_FILE") +INNER_EOF + + # Run Claude Code with the prompt (non-interactive) + cd /home/sentry/sentry && \ + claude --print --prompt-file /tmp/rca-prompt.txt \ + --allowedTools 'Bash(command:*)' 'Read' 'Glob' 'Grep' \ + --max-turns 50 \ + 2>&1 | tee /tmp/rca-agent-output.txt + + # Ensure the result file exists even if the agent didn't create it + if [ ! -f /tmp/rca-result.json ]; then + echo '{ + \"root_causes\": [{\"id\": 1, \"title\": \"Agent did not complete analysis\", \"description\": \"See agent output in workflow logs\", \"category\": \"unknown\", \"confidence\": \"low\", \"affected_runs\": [], \"affected_tests\": [], \"relevant_commits\": [], \"relevant_files\": [], \"suggested_fix\": \"Manual investigation required\", \"blast_radius\": \"unknown\"}], + \"action_items\": [{\"priority\": 1, \"action\": \"Review workflow logs manually\", \"rationale\": \"Automated analysis did not complete\", \"estimated_impact\": \"unknown\"}], + \"overall_confidence\": \"low\", + \"patterns\": {}, + \"full_analysis\": \"Agent did not produce structured output. Check workflow logs for partial analysis.\" + }' > /tmp/rca-result.json + fi + " || { + echo "WARNING: Agent execution failed, will use partial results" >&2 + } + +rm -f "$PROMPT_FILE" + +# ---------- Extract results from workspace ---------- + +echo "Extracting RCA results from workspace..." + +RCA_JSON=$(coder ssh "${WORKSPACE_NAME}" -- cat /tmp/rca-result.json 2>/dev/null || echo '{}') + +echo "Raw RCA JSON:" +echo "$RCA_JSON" | jq . 2>/dev/null || echo "$RCA_JSON" + +# ---------- Format outputs for GitHub Actions ---------- + +OVERALL_CONFIDENCE=$(echo "$RCA_JSON" | jq -r '.overall_confidence // "low"') + +# Format root causes as markdown +ROOT_CAUSES=$(echo "$RCA_JSON" | jq -r ' + (.root_causes // []) | to_entries | map( + .value | "#### \(.id // (.key + 1)). \(.title // "Unknown")\n" + + "- **Category**: `\(.category // "unknown")`\n" + + "- **Confidence**: `\(.confidence // "low")`\n" + + "- **Blast radius**: \(.blast_radius // "unknown")\n" + + "- **Description**: \(.description // "N/A")\n" + + "- **Suggested fix**: \(.suggested_fix // "N/A")\n" + ) | join("\n")') + +# Format action items as markdown +ACTION_ITEMS=$(echo "$RCA_JSON" | jq -r ' + (.action_items // []) | map( + "- **P\(.priority // "?")**: \(.action // "N/A") — _\(.rationale // "")_ (impact: \(.estimated_impact // "unknown"))" + ) | join("\n")') + +# Format full analysis +FULL_ANALYSIS=$(echo "$RCA_JSON" | jq -r '.full_analysis // "No detailed analysis available"') + +# Format patterns +PATTERNS=$(echo "$RCA_JSON" | jq -r ' + .patterns // {} | + "**Master broken**: \(.is_master_broken // "unknown")\n" + + "**Estimated flake rate**: \(.estimated_flake_rate // "unknown")\n" + + "**Failure timeline**: \(.failure_timeline // "N/A")\n" + + "**Most failing jobs**: \((.most_failing_jobs // {}) | to_entries | map("\(.key): \(.value)") | join(", "))\n" + + "**Most failing tests**: \((.most_failing_tests // {}) | to_entries | map("\(.key): \(.value)") | join(", "))"') + +# Build full RCA report markdown +RCA_REPORT=$(cat <> "$GITHUB_OUTPUT" + +echo "RCA analysis complete. Overall confidence: ${OVERALL_CONFIDENCE}" diff --git a/scripts/ci-rca/wait-for-workspace.sh b/scripts/ci-rca/wait-for-workspace.sh new file mode 100755 index 000000000000..c97a07d0fdf8 --- /dev/null +++ b/scripts/ci-rca/wait-for-workspace.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Wait for a Coder workspace to become ready (running + agent connected). +# +# Prerequisites: +# coder CLI must be installed and authenticated (via coder/setup-action) +# +# Required env vars: +# WORKSPACE_NAME - Workspace name to wait for + +# ---------- Wait for workspace to be running ---------- + +echo "Waiting for workspace '${WORKSPACE_NAME}' to be running..." +MAX_WAIT=300 +ELAPSED=0 +POLL_INTERVAL=10 + +while [ "$ELAPSED" -lt "$MAX_WAIT" ]; do + STATUS=$(coder show "${WORKSPACE_NAME}" --output json 2>/dev/null \ + | jq -r '.latest_build.status // "unknown"') + + echo " Status: ${STATUS} (${ELAPSED}s elapsed)" + + case "$STATUS" in + running) + echo "Workspace is running!" + break + ;; + failed|canceled) + echo "ERROR: Workspace build ${STATUS}" >&2 + exit 1 + ;; + esac + + sleep "$POLL_INTERVAL" + ELAPSED=$((ELAPSED + POLL_INTERVAL)) +done + +if [ "$ELAPSED" -ge "$MAX_WAIT" ]; then + echo "ERROR: Timed out waiting for workspace to start" >&2 + exit 1 +fi + +# ---------- Wait for agent to connect ---------- + +echo "Waiting for agent to connect..." +MAX_AGENT_WAIT=300 +AGENT_ELAPSED=0 + +while [ "$AGENT_ELAPSED" -lt "$MAX_AGENT_WAIT" ]; do + AGENT_STATUS=$(coder show "${WORKSPACE_NAME}" --output json 2>/dev/null \ + | jq -r '.latest_build.resources[]?.agents[]?.status // empty' | head -1) + + echo " Agent: ${AGENT_STATUS:-no agent yet} (${AGENT_ELAPSED}s elapsed)" + + if [ "$AGENT_STATUS" = "connected" ]; then + echo "Agent connected!" + break + fi + + sleep 10 + AGENT_ELAPSED=$((AGENT_ELAPSED + 10)) +done + +if [ "$AGENT_ELAPSED" -ge "$MAX_AGENT_WAIT" ]; then + echo "ERROR: Agent did not connect within timeout" >&2 + exit 1 +fi + +echo "Workspace is fully ready."