From 79ef85109ebeb22509979ead427d8627586cce11 Mon Sep 17 00:00:00 2001 From: Anatoli Tsikhamirau Date: Sat, 26 Sep 2026 00:01:31 +0200 Subject: [PATCH 1/2] feat(dynamic-labels): add a capacity-type label to choose spot or on-demand per job The label ghr-ec2-capacity-type:spot|on-demand overrides the capacity type of the runner configuration for one job. Other values are rejected as invalid runner labels. The label is ignored for dedicated hosts, and the max price is not passed to the fleet request for on-demand runners. --- docs/configuration.md | 22 +++++ .../ec2/src/control-plane/dynamic-labels.ts | 10 ++ .../ec2/src/control-plane/runner-creation.ts | 22 ++++- .../ec2/src/control-plane/scale-up.test.ts | 96 +++++++++++++++++++ .../aws/ec2/src/runners.d.ts | 1 + .../src/webhook/dynamic-labels-policy.test.ts | 7 ++ 6 files changed, 157 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2a8534e162..81ae07d5f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -560,6 +560,7 @@ When `use_dedicated_host = true`, runner instances are launched with `RunInstanc | Label | Description | Example value | EC2 Fleet | Dedicated host | | ------------------------------------ | -------------------------- | ---------------- | --------- | -------------- | | `ghr-ec2-instance-type:` | Set specific instance type | `c5.xlarge` | Yes | Yes | +| `ghr-ec2-capacity-type:` | Set the capacity type | `on-demand` | Yes | No | | `ghr-ec2-max-price:` | Set maximum spot price | `0.10` | Yes | No | | `ghr-ec2-subnet-id:` | Set subnet ID | `subnet-abc123` | Yes | Yes | | `ghr-ec2-availability-zone:` | Set availability zone | `us-east-1a` | Yes | Yes | @@ -568,6 +569,27 @@ When `use_dedicated_host = true`, runner instances are launched with `RunInstanc | `ghr-ec2-priority:` | Set launch priority | `1` | Yes | No | | `ghr-ec2-image-id:` | Override AMI ID | `ami-0abcdef123` | Yes | Yes | +##### Capacity type + +`ghr-ec2-capacity-type` selects spot or on-demand for the runner created for a job. It overrides `instance_target_capacity_type` of the runner configuration. Only `spot` and `on-demand` are accepted. Any other value is rejected and the job is not retried. + +- With `use_dedicated_host = true` the label is ignored, because `RunInstances` does not support spot. A `spot` value writes a warning to the log. +- `ghr-ec2-max-price` only applies to spot. It is not passed to the fleet request when the capacity type is `on-demand`. +- On-demand failover (`enable_on_demand_failover_for_errors`) follows the capacity type of the request. It applies to jobs that use spot and never to jobs that use on-demand. +- Tags for spot requests are defined in the launch template, and only when the default capacity type is spot without on-demand failover. A spot request created for a job that overrides the capacity type in another configuration does not get these tags. + +To allow only spot, restrict the key in the dynamic labels policy: + +```hcl +aws_dynamic_labels_policy = { + restricted_keys = { + "capacity-type" = { + allowed = ["spot"] + } + } +} +``` + ##### Instance Requirements — vCPU & Memory | Label | Description | Example value | EC2 Fleet | Dedicated host | diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts index 5bbc28a929..d0ad33fab0 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts @@ -33,6 +33,7 @@ import { InvalidRunnerLabelsError } from '../../../../core'; import { Ec2OverrideConfig } from '../runners.d'; const EC2_OVERRIDE_LIST_VALUE_SEPARATOR = ';'; +const CAPACITY_TYPES: string[] = ['spot', 'on-demand']; /** * Parses EC2 override configuration from GitHub labels. @@ -41,6 +42,7 @@ const EC2_OVERRIDE_LIST_VALUE_SEPARATOR = ';'; * * Basic Fleet Overrides: * - ghr-ec2-instance-type: - Set specific instance type (e.g., c5.xlarge) + * - ghr-ec2-capacity-type: - Set capacity type (spot or on-demand) * - ghr-ec2-max-price: - Set maximum spot price * - ghr-ec2-subnet-id: - Set subnet ID * - ghr-ec2-availability-zone: - Set availability zone @@ -148,6 +150,8 @@ export function parseEc2OverrideConfig( config.AvailabilityZone = value; } else if (key === 'availability-zone-id') { config.AvailabilityZoneId = value; + } else if (key === 'capacity-type') { + config.TargetCapacityType = value.toLowerCase() as Ec2OverrideConfig['TargetCapacityType']; } else if (key === 'max-price') { config.MaxPrice = value; } else if (key === 'priority') { @@ -338,6 +342,12 @@ export function validateEc2OverrideConfig(config: Ec2OverrideConfig): void { if (config.InstanceType && config.InstanceRequirements) { throw new InvalidRunnerLabelsError('InstanceType and InstanceRequirements cannot be used together'); } + + if (config.TargetCapacityType !== undefined && !CAPACITY_TYPES.includes(config.TargetCapacityType)) { + throw new InvalidRunnerLabelsError( + `Invalid capacity type '${config.TargetCapacityType}', expected one of: ${CAPACITY_TYPES.join(', ')}`, + ); + } } function splitEc2OverrideListValue(value: string): string[] { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 124b676ae6..30aee20cbf 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -62,6 +62,26 @@ export function loadEc2ProviderConfig(): Ec2ProviderConfig { }; } +function resolveCapacityType(config: CreateEC2RunnerConfig): CreateEC2RunnerConfig { + const { TargetCapacityType, ...overrides } = config.ec2OverrideConfig ?? {}; + let targetCapacityType = config.ec2instanceCriteria.targetCapacityType; + + if (TargetCapacityType && config.useDedicatedHost) { + if (TargetCapacityType === 'spot') { + logger.warn('Dedicated hosts do not support spot, the capacity type label is ignored.'); + } + } else if (TargetCapacityType) { + targetCapacityType = TargetCapacityType; + if (TargetCapacityType === 'on-demand') delete overrides.MaxPrice; + } + + return { + ...config, + ec2instanceCriteria: { ...config.ec2instanceCriteria, targetCapacityType }, + ec2OverrideConfig: Object.keys(overrides).length > 0 ? overrides : undefined, + }; +} + export async function createRunners( ec2Operations: Ec2RunnerResourceOperations, githubRunnerConfig: CreateGitHubRunnerConfig, @@ -74,7 +94,7 @@ export async function createRunners( ): Promise { let result: CreateRunnerResult; try { - const { scaleErrors, ...ec2CreateConfig } = ec2RunnerConfig; + const { scaleErrors, ...ec2CreateConfig } = resolveCapacityType(ec2RunnerConfig); const ec2Result = await ec2Operations.create({ ...ec2CreateConfig, runnerType: githubRunnerConfig.runnerType, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 2a79eea40d..eb571cddc6 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -412,6 +412,83 @@ describe('scaleUp with GHES', () => { ); }); + describe('capacity type label', () => { + it('creates an on-demand runner in a spot pool and keeps the label out of the fleet overrides', async () => { + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:on-demand'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'on-demand' }), + ec2OverrideConfig: undefined, + }), + ); + }); + + it('creates a spot runner in an on-demand pool', async () => { + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'on-demand'; + + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:spot'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'spot' }), + ec2OverrideConfig: undefined, + }), + ); + }); + + it('keeps the pool capacity type when the label is not set', async () => { + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-priority:1'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'spot' }), + }), + ); + }); + + it('does not pass the max price for an on-demand runner', async () => { + await createProviderRunners({ + labels: ['self-hosted', 'ghr-ec2-capacity-type:on-demand', 'ghr-ec2-max-price:0.50', 'ghr-ec2-priority:1'], + }); + + expect(mockCreateRunner).toHaveBeenCalledWith(expect.objectContaining({ ec2OverrideConfig: { Priority: 1 } })); + }); + + it('passes the max price for a spot runner', async () => { + await createProviderRunners({ + labels: ['self-hosted', 'ghr-ec2-capacity-type:spot', 'ghr-ec2-max-price:0.50'], + }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ ec2OverrideConfig: { MaxPrice: '0.50' } }), + ); + }); + + it('ignores the label for dedicated hosts', async () => { + process.env.USE_DEDICATED_HOST = 'true'; + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'on-demand'; + + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:spot'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + useDedicatedHost: true, + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'on-demand' }), + ec2OverrideConfig: undefined, + }), + ); + }); + + it('rejects an unsupported value before runner creation', async () => { + await expect( + createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:reserved'] }), + ).rejects.toThrow(InvalidRunnerLabelsError); + + expect(mockCreateRunner).not.toHaveBeenCalled(); + }); + }); + it('includes ec2OverrideConfig with priority and weighted capacity when specified', async () => { await createProviderRunners({ baseRunnerLabels: 'base-label', @@ -476,6 +553,16 @@ describe('scaleUp with GHES', () => { ).not.toThrow(); }); + it.each(['spot', 'on-demand'] as const)('accepts the %s capacity type', (TargetCapacityType) => { + expect(() => validateEc2OverrideConfig({ TargetCapacityType })).not.toThrow(); + }); + + it('rejects an unsupported capacity type', () => { + expect(() => validateEc2OverrideConfig({ TargetCapacityType: 'reserved' as unknown as 'spot' })).toThrow( + InvalidRunnerLabelsError, + ); + }); + it('rejects instance type with instance requirements', () => { expect(() => validateEc2OverrideConfig({ @@ -647,6 +734,15 @@ describe('parseEc2OverrideConfig', () => { expect(result?.AvailabilityZoneId).toBe('use1-az1'); }); + it('should parse capacity-type label', () => { + expect(parseEc2OverrideConfig(['ghr-ec2-capacity-type:spot'])?.TargetCapacityType).toBe('spot'); + expect(parseEc2OverrideConfig(['ghr-ec2-capacity-type:on-demand'])?.TargetCapacityType).toBe('on-demand'); + }); + + it('should parse capacity-type label case-insensitively', () => { + expect(parseEc2OverrideConfig(['ghr-ec2-capacity-type:On-Demand'])?.TargetCapacityType).toBe('on-demand'); + }); + it('should parse max-price label', () => { const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.50']); expect(result?.MaxPrice).toBe('0.50'); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts index e711ed5318..c4c76afe52 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts @@ -25,6 +25,7 @@ export interface Ec2OverrideConfig { InstanceRequirements?: InstanceRequirementsRequest; ImageId?: string; AvailabilityZoneId?: string; + TargetCapacityType?: 'spot' | 'on-demand'; } export interface RunnerInputParameters { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts index 0205606318..0e7c83fa08 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts @@ -42,6 +42,13 @@ describe('violationsAgainstPolicy', () => { expect(v[0].label).toBe('ghr-ec2-image-id:ami-1'); }); + it('restricts the capacity type to spot', () => { + const policy: Ec2DynamicLabelsPolicy = { restricted_keys: { 'capacity-type': { allowed: ['spot'] } } }; + const v = violationsAgainstPolicy(['ghr-ec2-capacity-type:spot', 'ghr-ec2-capacity-type:on-demand'], policy); + expect(v).toHaveLength(1); + expect(v[0].label).toBe('ghr-ec2-capacity-type:on-demand'); + }); + it('restricted key allowed glob with `*`', () => { const policy: Ec2DynamicLabelsPolicy = { restricted_keys: { 'instance-type': { allowed: ['m5.*', 'c5.*'] } } }; const v = violationsAgainstPolicy(['ghr-ec2-instance-type:m5.large', 'ghr-ec2-instance-type:r5.large'], policy); From bf4de9df2c4211b8eb3315e8e935d0f41f1d6065 Mon Sep 17 00:00:00 2001 From: Anatoli Tsikhamirau Date: Sun, 27 Sep 2026 19:30:11 +0200 Subject: [PATCH 2/2] refactor(dynamic-labels): make the EC2 config resolution step extensible resolveCapacityType is now one entry in a list of resolvers applied in order before runner creation, so a future per-job override can be added as another resolver without changing createRunners. --- .../aws/ec2/src/control-plane/runner-creation.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 30aee20cbf..a97dbb7a9b 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -62,6 +62,9 @@ export function loadEc2ProviderConfig(): Ec2ProviderConfig { }; } +// Extension point: add a resolver here for each per-job override that needs more than a plain merge. +type Ec2ConfigResolver = (config: CreateEC2RunnerConfig) => CreateEC2RunnerConfig; + function resolveCapacityType(config: CreateEC2RunnerConfig): CreateEC2RunnerConfig { const { TargetCapacityType, ...overrides } = config.ec2OverrideConfig ?? {}; let targetCapacityType = config.ec2instanceCriteria.targetCapacityType; @@ -82,6 +85,12 @@ function resolveCapacityType(config: CreateEC2RunnerConfig): CreateEC2RunnerConf }; } +const ec2ConfigResolvers: Ec2ConfigResolver[] = [resolveCapacityType]; + +function resolveEc2Config(config: CreateEC2RunnerConfig): CreateEC2RunnerConfig { + return ec2ConfigResolvers.reduce((resolvedConfig, resolve) => resolve(resolvedConfig), config); +} + export async function createRunners( ec2Operations: Ec2RunnerResourceOperations, githubRunnerConfig: CreateGitHubRunnerConfig, @@ -94,7 +103,7 @@ export async function createRunners( ): Promise { let result: CreateRunnerResult; try { - const { scaleErrors, ...ec2CreateConfig } = resolveCapacityType(ec2RunnerConfig); + const { scaleErrors, ...ec2CreateConfig } = resolveEc2Config(ec2RunnerConfig); const ec2Result = await ec2Operations.create({ ...ec2CreateConfig, runnerType: githubRunnerConfig.runnerType,