diff --git a/lib/asg-runner-stack.ts b/lib/asg-runner-stack.ts index 567c4d7..bda36ae 100644 --- a/lib/asg-runner-stack.ts +++ b/lib/asg-runner-stack.ts @@ -238,10 +238,23 @@ export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack { ] }; + // Dedicated-host runners (macOS/Windows) run on scarce metal capacity that is + // allocated just-in-time. A rolling update that terminates an instance BEFORE its + // replacement is InService leaves a capacity gap: if metal capacity is momentarily + // unavailable, the replacement cannot launch, the batch never stabilizes, and the + // CloudFormation update fails (and can wedge in UPDATE_ROLLBACK_FAILED). To make + // deploys resilient we give these ASGs N+1 headroom (maxCapacity = desired + 1) and + // keep desired instances in service during a rolling update (launch-before-terminate), + // which requires a reserved spare host slot (see the host resource group below, which + // retains hosts rather than auto-releasing them). maxCapacity > desired also means a + // bad instance can always be replaced without deadlocking. + const usesDedicatedHosts = this.requiresDedicatedHosts(); + const maxCapacity = usesDedicatedHosts ? props.type.desiredInstances + 1 : props.type.desiredInstances; + const asg = new autoscaling.AutoScalingGroup(this, asgName, { vpc, desiredCapacity: props.type.desiredInstances, - maxCapacity: props.type.desiredInstances, + maxCapacity, minCapacity: 0, healthCheck: autoscaling.HealthCheck.ec2({ grace: cdk.Duration.seconds(3600) @@ -251,7 +264,13 @@ export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack { // Defaults shown here explicitly except for pauseTime // and minSuccesPercentage maxBatchSize: 1, - minInstancesInService: 0, + // For dedicated-host (metal) runners, keep all desired instances in service while + // rolling so a replacement must reach InService before the old instance is + // terminated (launch-before-terminate). This avoids the "terminate first, then + // fail to launch on insufficient metal capacity" wedge. Requires N+1 capacity + // (maxCapacity above) and a reserved host slot. Non-metal ASGs keep the prior + // behavior (0), since they scale on abundant capacity. + minInstancesInService: usesDedicatedHosts ? props.type.desiredInstances : 0, suspendProcesses: [ autoscaling.ScalingProcess.HEALTH_CHECK, autoscaling.ScalingProcess.REPLACE_UNHEALTHY, @@ -293,8 +312,17 @@ export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack { values: ['true'] }, { + // Retain dedicated hosts instead of auto-releasing them after an instance + // terminates. macOS/Windows metal capacity is scarce and allocated + // just-in-time; auto-releasing forces the ASG to re-compete for metal + // capacity on every deploy/replacement, which causes "Insufficient + // capacity" launch failures and wedged CloudFormation updates. Retaining + // hosts keeps the (N+1) reserved slots so launch-before-terminate rolling + // updates always have somewhere to place the replacement. Note: macOS + // dedicated hosts have a ~24h minimum allocation regardless, so retaining + // them adds little effective cost while removing the capacity gamble. name: 'auto-release-host', - values: ['true'] + values: ['false'] }, { name: 'any-host-based-license-configuration', diff --git a/test/asg-runner-stack.test.ts b/test/asg-runner-stack.test.ts index 018cb70..ca6b26f 100644 --- a/test/asg-runner-stack.test.ts +++ b/test/asg-runner-stack.test.ts @@ -1,5 +1,5 @@ import * as cdk from 'aws-cdk-lib'; -import { Template } from 'aws-cdk-lib/assertions'; +import { Match, Template } from 'aws-cdk-lib/assertions'; import { PlatformType, RunnerConfig, RunnerType } from '../config/runner-config'; import { ASGRunnerStack } from '../lib/asg-runner-stack'; import { ENVIRONMENT_STAGE } from '../lib/finch-pipeline-app-stage'; @@ -82,4 +82,54 @@ describe('ASGRunnerStack test', () => { expect(stack.terminationProtection).toBeTruthy(); }); }); + + it('gives dedicated-host runners N+1 capacity and launch-before-terminate', () => { + // macOS/Windows runners run on scarce dedicated-host (metal) capacity. To avoid a + // rolling update terminating an instance before its replacement can be placed, they + // get maxCapacity = desired + 1 and keep desired instances in service during rollout. + runnerConfig.runnerTypes + .filter((type) => type.platform === PlatformType.MAC || type.platform === PlatformType.WINDOWS) + .forEach((type) => { + const stack = stacks.find((s) => s.stackName === generateASGStackName(type)); + const template = Template.fromStack(stack!); + template.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', { + MinSize: '0', + MaxSize: `${type.desiredInstances + 1}`, + DesiredCapacity: `${type.desiredInstances}` + }); + // Rolling update keeps desired instances in service (launch-before-terminate). + template.hasResource('AWS::AutoScaling::AutoScalingGroup', { + UpdatePolicy: { + AutoScalingRollingUpdate: { + MinInstancesInService: type.desiredInstances, + MaxBatchSize: 1 + } + } + }); + // Dedicated hosts are retained (not auto-released) so the reserved slot persists. + template.hasResourceProperties('AWS::ResourceGroups::Group', { + Configuration: Match.arrayWith([ + Match.objectLike({ + Type: 'AWS::EC2::HostManagement', + Parameters: Match.arrayWith([ + Match.objectLike({ Name: 'auto-release-host', Values: ['false'] }) + ]) + }) + ]) + }); + }); + }); + + it('non-dedicated-host runners keep maxCapacity equal to desired', () => { + runnerConfig.runnerTypes + .filter((type) => type.platform === PlatformType.AMAZONLINUX) + .forEach((type) => { + const stack = stacks.find((s) => s.stackName === generateASGStackName(type)); + const template = Template.fromStack(stack!); + template.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', { + MaxSize: `${type.desiredInstances}`, + DesiredCapacity: `${type.desiredInstances}` + }); + }); + }); });