diff --git a/docs/configuration.md b/docs/configuration.md index 2a8534e162..81ae07d5f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -560,6 +560,7 @@ When `use_dedicated_host = true`, runner instances are launched with `RunInstanc | Label | Description | Example value | EC2 Fleet | Dedicated host | | ------------------------------------ | -------------------------- | ---------------- | --------- | -------------- | | `ghr-ec2-instance-type:` | Set specific instance type | `c5.xlarge` | Yes | Yes | +| `ghr-ec2-capacity-type:` | Set the capacity type | `on-demand` | Yes | No | | `ghr-ec2-max-price:` | Set maximum spot price | `0.10` | Yes | No | | `ghr-ec2-subnet-id:` | Set subnet ID | `subnet-abc123` | Yes | Yes | | `ghr-ec2-availability-zone:` | Set availability zone | `us-east-1a` | Yes | Yes | @@ -568,6 +569,27 @@ When `use_dedicated_host = true`, runner instances are launched with `RunInstanc | `ghr-ec2-priority:` | Set launch priority | `1` | Yes | No | | `ghr-ec2-image-id:` | Override AMI ID | `ami-0abcdef123` | Yes | Yes | +##### Capacity type + +`ghr-ec2-capacity-type` selects spot or on-demand for the runner created for a job. It overrides `instance_target_capacity_type` of the runner configuration. Only `spot` and `on-demand` are accepted. Any other value is rejected and the job is not retried. + +- With `use_dedicated_host = true` the label is ignored, because `RunInstances` does not support spot. A `spot` value writes a warning to the log. +- `ghr-ec2-max-price` only applies to spot. It is not passed to the fleet request when the capacity type is `on-demand`. +- On-demand failover (`enable_on_demand_failover_for_errors`) follows the capacity type of the request. It applies to jobs that use spot and never to jobs that use on-demand. +- Tags for spot requests are defined in the launch template, and only when the default capacity type is spot without on-demand failover. A spot request created for a job that overrides the capacity type in another configuration does not get these tags. + +To allow only spot, restrict the key in the dynamic labels policy: + +```hcl +aws_dynamic_labels_policy = { + restricted_keys = { + "capacity-type" = { + allowed = ["spot"] + } + } +} +``` + ##### Instance Requirements — vCPU & Memory | Label | Description | Example value | EC2 Fleet | Dedicated host | diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts index 5bbc28a929..d0ad33fab0 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts @@ -33,6 +33,7 @@ import { InvalidRunnerLabelsError } from '../../../../core'; import { Ec2OverrideConfig } from '../runners.d'; const EC2_OVERRIDE_LIST_VALUE_SEPARATOR = ';'; +const CAPACITY_TYPES: string[] = ['spot', 'on-demand']; /** * Parses EC2 override configuration from GitHub labels. @@ -41,6 +42,7 @@ const EC2_OVERRIDE_LIST_VALUE_SEPARATOR = ';'; * * Basic Fleet Overrides: * - ghr-ec2-instance-type: - Set specific instance type (e.g., c5.xlarge) + * - ghr-ec2-capacity-type: - Set capacity type (spot or on-demand) * - ghr-ec2-max-price: - Set maximum spot price * - ghr-ec2-subnet-id: - Set subnet ID * - ghr-ec2-availability-zone: - Set availability zone @@ -148,6 +150,8 @@ export function parseEc2OverrideConfig( config.AvailabilityZone = value; } else if (key === 'availability-zone-id') { config.AvailabilityZoneId = value; + } else if (key === 'capacity-type') { + config.TargetCapacityType = value.toLowerCase() as Ec2OverrideConfig['TargetCapacityType']; } else if (key === 'max-price') { config.MaxPrice = value; } else if (key === 'priority') { @@ -338,6 +342,12 @@ export function validateEc2OverrideConfig(config: Ec2OverrideConfig): void { if (config.InstanceType && config.InstanceRequirements) { throw new InvalidRunnerLabelsError('InstanceType and InstanceRequirements cannot be used together'); } + + if (config.TargetCapacityType !== undefined && !CAPACITY_TYPES.includes(config.TargetCapacityType)) { + throw new InvalidRunnerLabelsError( + `Invalid capacity type '${config.TargetCapacityType}', expected one of: ${CAPACITY_TYPES.join(', ')}`, + ); + } } function splitEc2OverrideListValue(value: string): string[] { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 124b676ae6..a97dbb7a9b 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -62,6 +62,35 @@ export function loadEc2ProviderConfig(): Ec2ProviderConfig { }; } +// Extension point: add a resolver here for each per-job override that needs more than a plain merge. +type Ec2ConfigResolver = (config: CreateEC2RunnerConfig) => CreateEC2RunnerConfig; + +function resolveCapacityType(config: CreateEC2RunnerConfig): CreateEC2RunnerConfig { + const { TargetCapacityType, ...overrides } = config.ec2OverrideConfig ?? {}; + let targetCapacityType = config.ec2instanceCriteria.targetCapacityType; + + if (TargetCapacityType && config.useDedicatedHost) { + if (TargetCapacityType === 'spot') { + logger.warn('Dedicated hosts do not support spot, the capacity type label is ignored.'); + } + } else if (TargetCapacityType) { + targetCapacityType = TargetCapacityType; + if (TargetCapacityType === 'on-demand') delete overrides.MaxPrice; + } + + return { + ...config, + ec2instanceCriteria: { ...config.ec2instanceCriteria, targetCapacityType }, + ec2OverrideConfig: Object.keys(overrides).length > 0 ? overrides : undefined, + }; +} + +const ec2ConfigResolvers: Ec2ConfigResolver[] = [resolveCapacityType]; + +function resolveEc2Config(config: CreateEC2RunnerConfig): CreateEC2RunnerConfig { + return ec2ConfigResolvers.reduce((resolvedConfig, resolve) => resolve(resolvedConfig), config); +} + export async function createRunners( ec2Operations: Ec2RunnerResourceOperations, githubRunnerConfig: CreateGitHubRunnerConfig, @@ -74,7 +103,7 @@ export async function createRunners( ): Promise { let result: CreateRunnerResult; try { - const { scaleErrors, ...ec2CreateConfig } = ec2RunnerConfig; + const { scaleErrors, ...ec2CreateConfig } = resolveEc2Config(ec2RunnerConfig); const ec2Result = await ec2Operations.create({ ...ec2CreateConfig, runnerType: githubRunnerConfig.runnerType, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 2a79eea40d..eb571cddc6 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -412,6 +412,83 @@ describe('scaleUp with GHES', () => { ); }); + describe('capacity type label', () => { + it('creates an on-demand runner in a spot pool and keeps the label out of the fleet overrides', async () => { + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:on-demand'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'on-demand' }), + ec2OverrideConfig: undefined, + }), + ); + }); + + it('creates a spot runner in an on-demand pool', async () => { + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'on-demand'; + + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:spot'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'spot' }), + ec2OverrideConfig: undefined, + }), + ); + }); + + it('keeps the pool capacity type when the label is not set', async () => { + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-priority:1'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'spot' }), + }), + ); + }); + + it('does not pass the max price for an on-demand runner', async () => { + await createProviderRunners({ + labels: ['self-hosted', 'ghr-ec2-capacity-type:on-demand', 'ghr-ec2-max-price:0.50', 'ghr-ec2-priority:1'], + }); + + expect(mockCreateRunner).toHaveBeenCalledWith(expect.objectContaining({ ec2OverrideConfig: { Priority: 1 } })); + }); + + it('passes the max price for a spot runner', async () => { + await createProviderRunners({ + labels: ['self-hosted', 'ghr-ec2-capacity-type:spot', 'ghr-ec2-max-price:0.50'], + }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ ec2OverrideConfig: { MaxPrice: '0.50' } }), + ); + }); + + it('ignores the label for dedicated hosts', async () => { + process.env.USE_DEDICATED_HOST = 'true'; + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'on-demand'; + + await createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:spot'] }); + + expect(mockCreateRunner).toHaveBeenCalledWith( + expect.objectContaining({ + useDedicatedHost: true, + ec2instanceCriteria: expect.objectContaining({ targetCapacityType: 'on-demand' }), + ec2OverrideConfig: undefined, + }), + ); + }); + + it('rejects an unsupported value before runner creation', async () => { + await expect( + createProviderRunners({ labels: ['self-hosted', 'ghr-ec2-capacity-type:reserved'] }), + ).rejects.toThrow(InvalidRunnerLabelsError); + + expect(mockCreateRunner).not.toHaveBeenCalled(); + }); + }); + it('includes ec2OverrideConfig with priority and weighted capacity when specified', async () => { await createProviderRunners({ baseRunnerLabels: 'base-label', @@ -476,6 +553,16 @@ describe('scaleUp with GHES', () => { ).not.toThrow(); }); + it.each(['spot', 'on-demand'] as const)('accepts the %s capacity type', (TargetCapacityType) => { + expect(() => validateEc2OverrideConfig({ TargetCapacityType })).not.toThrow(); + }); + + it('rejects an unsupported capacity type', () => { + expect(() => validateEc2OverrideConfig({ TargetCapacityType: 'reserved' as unknown as 'spot' })).toThrow( + InvalidRunnerLabelsError, + ); + }); + it('rejects instance type with instance requirements', () => { expect(() => validateEc2OverrideConfig({ @@ -647,6 +734,15 @@ describe('parseEc2OverrideConfig', () => { expect(result?.AvailabilityZoneId).toBe('use1-az1'); }); + it('should parse capacity-type label', () => { + expect(parseEc2OverrideConfig(['ghr-ec2-capacity-type:spot'])?.TargetCapacityType).toBe('spot'); + expect(parseEc2OverrideConfig(['ghr-ec2-capacity-type:on-demand'])?.TargetCapacityType).toBe('on-demand'); + }); + + it('should parse capacity-type label case-insensitively', () => { + expect(parseEc2OverrideConfig(['ghr-ec2-capacity-type:On-Demand'])?.TargetCapacityType).toBe('on-demand'); + }); + it('should parse max-price label', () => { const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.50']); expect(result?.MaxPrice).toBe('0.50'); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts index e711ed5318..c4c76afe52 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts @@ -25,6 +25,7 @@ export interface Ec2OverrideConfig { InstanceRequirements?: InstanceRequirementsRequest; ImageId?: string; AvailabilityZoneId?: string; + TargetCapacityType?: 'spot' | 'on-demand'; } export interface RunnerInputParameters { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts index 0205606318..0e7c83fa08 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts @@ -42,6 +42,13 @@ describe('violationsAgainstPolicy', () => { expect(v[0].label).toBe('ghr-ec2-image-id:ami-1'); }); + it('restricts the capacity type to spot', () => { + const policy: Ec2DynamicLabelsPolicy = { restricted_keys: { 'capacity-type': { allowed: ['spot'] } } }; + const v = violationsAgainstPolicy(['ghr-ec2-capacity-type:spot', 'ghr-ec2-capacity-type:on-demand'], policy); + expect(v).toHaveLength(1); + expect(v[0].label).toBe('ghr-ec2-capacity-type:on-demand'); + }); + it('restricted key allowed glob with `*`', () => { const policy: Ec2DynamicLabelsPolicy = { restricted_keys: { 'instance-type': { allowed: ['m5.*', 'c5.*'] } } }; const v = violationsAgainstPolicy(['ghr-ec2-instance-type:m5.large', 'ghr-ec2-instance-type:r5.large'], policy);