diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts index c4ff1e5d76..3879cc2f6e 100644 --- a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts @@ -270,6 +270,59 @@ describe(`Test job retry check`, () => { // assert expect(publishMessage).not.toHaveBeenCalled(); }); + + it(`should still publish a retry and record the metric when isJobQueued throws a transient error (fail-open)`, async () => { + // setup + mockOctokit.actions.getJobForWorkflowRun.mockRejectedValue(new Error('GitHub API 502')); + + const message: ActionRequestMessageRetry = { + eventType: 'workflow_job', + id: 0, + installationId: 0, + repositoryName: 'test', + repositoryOwner: 'github-aws-runners', + repoOwnerType: 'Organization', + retryCounter: 0, + }; + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENVIRONMENT = 'test'; + process.env.RUNNER_NAME_PREFIX = 'test'; + process.env.ENABLE_METRIC_JOB_RETRY = 'true'; + process.env.JOB_QUEUE_SCALE_UP_URL = + 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue'; + + // act + await checkAndRetryJob(message); + + // assert + expect(publishMessage).toHaveBeenCalledWith( + JSON.stringify({ ...message }), + 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue', + ); + expect(createSingleMetric).toHaveBeenCalled(); + }); + + it(`should not publish a retry when the event type is unsupported`, async () => { + const message = { + eventType: 'check_run', + id: 0, + installationId: 0, + repositoryName: 'test', + repositoryOwner: 'github-aws-runners', + repoOwnerType: 'Organization', + retryCounter: 0, + } as unknown as ActionRequestMessageRetry; + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'test'; + process.env.JOB_QUEUE_SCALE_UP_URL = + 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue'; + + // act + await checkAndRetryJob(message); + + // assert + expect(publishMessage).not.toHaveBeenCalled(); + }); }); describe('Test job retry handler (batch processing)', () => { diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.ts index 8f7d6e2289..c978b54380 100644 --- a/lambdas/functions/control-plane/src/scale-runners/job-retry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.ts @@ -1,6 +1,6 @@ import { addPersistentContextToChildLogger, createSingleMetric, logger } from '@aws-github-runner/aws-powertools-util'; import { publishMessage } from '../aws/sqs'; -import { getGitHubEnterpriseApiUrl, isJobQueued } from './github-runner'; +import { getGitHubEnterpriseApiUrl, isJobQueued, UnsupportedEventError } from './github-runner'; import type { ActionRequestMessage, ActionRequestMessageRetry } from './types'; import { getOctokit } from '../github/octokit'; import { MetricUnit } from '@aws-lambda-powertools/metrics'; @@ -63,8 +63,22 @@ export async function checkAndRetryJob(payload: ActionRequestMessageRetry): Prom const { ghesApiUrl } = getGitHubEnterpriseApiUrl(); const ghClient = await getOctokit(ghesApiUrl, enableOrgLevel, payload); - // check job is still queued - if (await isJobQueued(ghClient, payload)) { + let jobQueued = true; + try { + jobQueued = await isJobQueued(ghClient, payload); + } catch (e) { + if (e instanceof UnsupportedEventError) { + logger.debug(`Unsupported event type, skipping retry`, { payload }); + return; + } + const err = e as Error & { status?: number }; + logger.warn('isJobQueued check failed, assuming job is still queued (fail-open)', { + error: err.message, + status: err.status, + }); + } + + if (jobQueued) { await publishMessage(JSON.stringify(payload), jobQueueUrl); createMetric(enableMetrics, environment, payload); logger.info(`Job is still queued, message published to build queue and will be handled by scale-up.`, { payload });