From 73d873525a6c0eb3202e29142b9d10dc3fc5ec7f Mon Sep 17 00:00:00 2001
From: Anatoli Tsikhamirau <tihomirovanatoliy@gmail.com>
Date: Sat, 12 Sep 2026 21:08:21 +0200
Subject: [PATCH] fix(job-retry): fail open instead of dropping the job when
 isJobQueued errors

checkAndRetryJob had no error handling around isJobQueued: any error
from it (a transient GitHub API failure, a rate limit, an unsupported
event type) propagated out of the function. The lambda handler catches
and only logs that error, so the SQS message is still marked processed
and never redelivered - the job's retry is silently dropped for good,
and the RetryJob metric for that attempt is never recorded either.

isJobQueued now runs in a try/catch that mirrors the same check in
scale-up.ts: on an UnsupportedEventError the retry is skipped (that
error can never resolve itself), on any other error the job is assumed
still queued and the retry is published anyway, since a transient
error is not evidence the job stopped needing a runner.
---
 .../src/scale-runners/job-retry.test.ts       | 53 +++++++++++++++++++
 .../src/scale-runners/job-retry.ts            | 20 +++++--
 2 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts
index c4ff1e5d76..3879cc2f6e 100644
--- a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts
@@ -270,6 +270,59 @@ describe(`Test job retry check`, () => {
     // assert
     expect(publishMessage).not.toHaveBeenCalled();
   });
+
+  it(`should still publish a retry and record the metric when isJobQueued throws a transient error (fail-open)`, async () => {
+    // setup
+    mockOctokit.actions.getJobForWorkflowRun.mockRejectedValue(new Error('GitHub API 502'));
+
+    const message: ActionRequestMessageRetry = {
+      eventType: 'workflow_job',
+      id: 0,
+      installationId: 0,
+      repositoryName: 'test',
+      repositoryOwner: 'github-aws-runners',
+      repoOwnerType: 'Organization',
+      retryCounter: 0,
+    };
+    process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+    process.env.ENVIRONMENT = 'test';
+    process.env.RUNNER_NAME_PREFIX = 'test';
+    process.env.ENABLE_METRIC_JOB_RETRY = 'true';
+    process.env.JOB_QUEUE_SCALE_UP_URL =
+      'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue';
+
+    // act
+    await checkAndRetryJob(message);
+
+    // assert
+    expect(publishMessage).toHaveBeenCalledWith(
+      JSON.stringify({ ...message }),
+      'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue',
+    );
+    expect(createSingleMetric).toHaveBeenCalled();
+  });
+
+  it(`should not publish a retry when the event type is unsupported`, async () => {
+    const message = {
+      eventType: 'check_run',
+      id: 0,
+      installationId: 0,
+      repositoryName: 'test',
+      repositoryOwner: 'github-aws-runners',
+      repoOwnerType: 'Organization',
+      retryCounter: 0,
+    } as unknown as ActionRequestMessageRetry;
+    process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+    process.env.RUNNER_NAME_PREFIX = 'test';
+    process.env.JOB_QUEUE_SCALE_UP_URL =
+      'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue';
+
+    // act
+    await checkAndRetryJob(message);
+
+    // assert
+    expect(publishMessage).not.toHaveBeenCalled();
+  });
 });
 
 describe('Test job retry handler (batch processing)', () => {
diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.ts
index 8f7d6e2289..c978b54380 100644
--- a/lambdas/functions/control-plane/src/scale-runners/job-retry.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.ts
@@ -1,6 +1,6 @@
 import { addPersistentContextToChildLogger, createSingleMetric, logger } from '@aws-github-runner/aws-powertools-util';
 import { publishMessage } from '../aws/sqs';
-import { getGitHubEnterpriseApiUrl, isJobQueued } from './github-runner';
+import { getGitHubEnterpriseApiUrl, isJobQueued, UnsupportedEventError } from './github-runner';
 import type { ActionRequestMessage, ActionRequestMessageRetry } from './types';
 import { getOctokit } from '../github/octokit';
 import { MetricUnit } from '@aws-lambda-powertools/metrics';
@@ -63,8 +63,22 @@ export async function checkAndRetryJob(payload: ActionRequestMessageRetry): Prom
   const { ghesApiUrl } = getGitHubEnterpriseApiUrl();
   const ghClient = await getOctokit(ghesApiUrl, enableOrgLevel, payload);
 
-  // check job is still queued
-  if (await isJobQueued(ghClient, payload)) {
+  let jobQueued = true;
+  try {
+    jobQueued = await isJobQueued(ghClient, payload);
+  } catch (e) {
+    if (e instanceof UnsupportedEventError) {
+      logger.debug(`Unsupported event type, skipping retry`, { payload });
+      return;
+    }
+    const err = e as Error & { status?: number };
+    logger.warn('isJobQueued check failed, assuming job is still queued (fail-open)', {
+      error: err.message,
+      status: err.status,
+    });
+  }
+
+  if (jobQueued) {
     await publishMessage(JSON.stringify(payload), jobQueueUrl);
     createMetric(enableMetrics, environment, payload);
     logger.info(`Job is still queued, message published to build queue and will be handled by scale-up.`, { payload });