fix(job-retry): fail open instead of dropping the job when isJobQueued errors

lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts

@@ -270,6 +270,59 @@ describe(`Test job retry check`, () => {
     // assert
     expect(publishMessage).not.toHaveBeenCalled();
   });
+
+  it(`should still publish a retry and record the metric when isJobQueued throws a transient error (fail-open)`, async () => {
+    // setup
+    mockOctokit.actions.getJobForWorkflowRun.mockRejectedValue(new Error('GitHub API 502'));
+
+    const message: ActionRequestMessageRetry = {
+      eventType: 'workflow_job',
+      id: 0,
+      installationId: 0,
+      repositoryName: 'test',
+      repositoryOwner: 'github-aws-runners',
+      repoOwnerType: 'Organization',
+      retryCounter: 0,
+    };
+    process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+    process.env.ENVIRONMENT = 'test';
+    process.env.RUNNER_NAME_PREFIX = 'test';
+    process.env.ENABLE_METRIC_JOB_RETRY = 'true';
+    process.env.JOB_QUEUE_SCALE_UP_URL =
+      'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue';
+
+    // act
+    await checkAndRetryJob(message);
+
+    // assert
+    expect(publishMessage).toHaveBeenCalledWith(
+      JSON.stringify({ ...message }),
+      'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue',
+    );
+    expect(createSingleMetric).toHaveBeenCalled();
+  });
+
+  it(`should not publish a retry when the event type is unsupported`, async () => {
+    const message = {
+      eventType: 'check_run',
+      id: 0,
+      installationId: 0,
+      repositoryName: 'test',
+      repositoryOwner: 'github-aws-runners',
+      repoOwnerType: 'Organization',
+      retryCounter: 0,
+    } as unknown as ActionRequestMessageRetry;
+    process.env.ENABLE_ORGANIZATION_RUNNERS = 'true';
+    process.env.RUNNER_NAME_PREFIX = 'test';
+    process.env.JOB_QUEUE_SCALE_UP_URL =
+      'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue';
+
+    // act
+    await checkAndRetryJob(message);
+
+    // assert
+    expect(publishMessage).not.toHaveBeenCalled();
+  });
 });
 
 describe('Test job retry handler (batch processing)', () => {

lambdas/functions/control-plane/src/scale-runners/job-retry.ts

@@ -1,6 +1,6 @@
 import { addPersistentContextToChildLogger, createSingleMetric, logger } from '@aws-github-runner/aws-powertools-util';
 import { publishMessage } from '../aws/sqs';
-import { getGitHubEnterpriseApiUrl, isJobQueued } from './github-runner';
+import { getGitHubEnterpriseApiUrl, isJobQueued, UnsupportedEventError } from './github-runner';
 import type { ActionRequestMessage, ActionRequestMessageRetry } from './types';
 import { getOctokit } from '../github/octokit';
 import { MetricUnit } from '@aws-lambda-powertools/metrics';
@@ -63,8 +63,22 @@ export async function checkAndRetryJob(payload: ActionRequestMessageRetry): Prom
   const { ghesApiUrl } = getGitHubEnterpriseApiUrl();
   const ghClient = await getOctokit(ghesApiUrl, enableOrgLevel, payload);
 
-  // check job is still queued
-  if (await isJobQueued(ghClient, payload)) {
+  let jobQueued = true;
+  try {
+    jobQueued = await isJobQueued(ghClient, payload);
+  } catch (e) {
+    if (e instanceof UnsupportedEventError) {
+      logger.debug(`Unsupported event type, skipping retry`, { payload });
+      return;
+    }
+    const err = e as Error & { status?: number };
+    logger.warn('isJobQueued check failed, assuming job is still queued (fail-open)', {
+      error: err.message,
+      status: err.status,
+    });
+  }
+
+  if (jobQueued) {
     await publishMessage(JSON.stringify(payload), jobQueueUrl);
     createMetric(enableMetrics, environment, payload);
     logger.info(`Job is still queued, message published to build queue and will be handled by scale-up.`, { payload });