fix(runners): make dedicated-host (mac/windows) ASG deploys resilient to metal capacity

lib/asg-runner-stack.ts

@@ -238,10 +238,23 @@ export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack {
       ]
     };
 
+    // Dedicated-host runners (macOS/Windows) run on scarce metal capacity that is
+    // allocated just-in-time. A rolling update that terminates an instance BEFORE its
+    // replacement is InService leaves a capacity gap: if metal capacity is momentarily
+    // unavailable, the replacement cannot launch, the batch never stabilizes, and the
+    // CloudFormation update fails (and can wedge in UPDATE_ROLLBACK_FAILED). To make
+    // deploys resilient we give these ASGs N+1 headroom (maxCapacity = desired + 1) and
+    // keep desired instances in service during a rolling update (launch-before-terminate),
+    // which requires a reserved spare host slot (see the host resource group below, which
+    // retains hosts rather than auto-releasing them). maxCapacity > desired also means a
+    // bad instance can always be replaced without deadlocking.
+    const usesDedicatedHosts = this.requiresDedicatedHosts();
+    const maxCapacity = usesDedicatedHosts ? props.type.desiredInstances + 1 : props.type.desiredInstances;
+
     const asg = new autoscaling.AutoScalingGroup(this, asgName, {
       vpc,
       desiredCapacity: props.type.desiredInstances,
-      maxCapacity: props.type.desiredInstances,
+      maxCapacity,
       minCapacity: 0,
       healthCheck: autoscaling.HealthCheck.ec2({
         grace: cdk.Duration.seconds(3600)
@@ -251,7 +264,13 @@ export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack {
         // Defaults shown here explicitly except for pauseTime
         // and minSuccesPercentage
         maxBatchSize: 1,
-        minInstancesInService: 0,
+        // For dedicated-host (metal) runners, keep all desired instances in service while
+        // rolling so a replacement must reach InService before the old instance is
+        // terminated (launch-before-terminate). This avoids the "terminate first, then
+        // fail to launch on insufficient metal capacity" wedge. Requires N+1 capacity
+        // (maxCapacity above) and a reserved host slot. Non-metal ASGs keep the prior
+        // behavior (0), since they scale on abundant capacity.
+        minInstancesInService: usesDedicatedHosts ? props.type.desiredInstances : 0,
         suspendProcesses: [
           autoscaling.ScalingProcess.HEALTH_CHECK,
           autoscaling.ScalingProcess.REPLACE_UNHEALTHY,
@@ -293,8 +312,17 @@ export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack {
               values: ['true']
             },
             {
+              // Retain dedicated hosts instead of auto-releasing them after an instance
+              // terminates. macOS/Windows metal capacity is scarce and allocated
+              // just-in-time; auto-releasing forces the ASG to re-compete for metal
+              // capacity on every deploy/replacement, which causes "Insufficient
+              // capacity" launch failures and wedged CloudFormation updates. Retaining
+              // hosts keeps the (N+1) reserved slots so launch-before-terminate rolling
+              // updates always have somewhere to place the replacement. Note: macOS
+              // dedicated hosts have a ~24h minimum allocation regardless, so retaining
+              // them adds little effective cost while removing the capacity gamble.
               name: 'auto-release-host',
-              values: ['true']
+              values: ['false']
             },
             {
               name: 'any-host-based-license-configuration',

test/asg-runner-stack.test.ts

@@ -1,5 +1,5 @@
 import * as cdk from 'aws-cdk-lib';
-import { Template } from 'aws-cdk-lib/assertions';
+import { Match, Template } from 'aws-cdk-lib/assertions';
 import { PlatformType, RunnerConfig, RunnerType } from '../config/runner-config';
 import { ASGRunnerStack } from '../lib/asg-runner-stack';
 import { ENVIRONMENT_STAGE } from '../lib/finch-pipeline-app-stage';
@@ -82,4 +82,54 @@ describe('ASGRunnerStack test', () => {
       expect(stack.terminationProtection).toBeTruthy();
     });
   });
+
+  it('gives dedicated-host runners N+1 capacity and launch-before-terminate', () => {
+    // macOS/Windows runners run on scarce dedicated-host (metal) capacity. To avoid a
+    // rolling update terminating an instance before its replacement can be placed, they
+    // get maxCapacity = desired + 1 and keep desired instances in service during rollout.
+    runnerConfig.runnerTypes
+      .filter((type) => type.platform === PlatformType.MAC || type.platform === PlatformType.WINDOWS)
+      .forEach((type) => {
+        const stack = stacks.find((s) => s.stackName === generateASGStackName(type));
+        const template = Template.fromStack(stack!);
+        template.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', {
+          MinSize: '0',
+          MaxSize: `${type.desiredInstances + 1}`,
+          DesiredCapacity: `${type.desiredInstances}`
+        });
+        // Rolling update keeps desired instances in service (launch-before-terminate).
+        template.hasResource('AWS::AutoScaling::AutoScalingGroup', {
+          UpdatePolicy: {
+            AutoScalingRollingUpdate: {
+              MinInstancesInService: type.desiredInstances,
+              MaxBatchSize: 1
+            }
+          }
+        });
+        // Dedicated hosts are retained (not auto-released) so the reserved slot persists.
+        template.hasResourceProperties('AWS::ResourceGroups::Group', {
+          Configuration: Match.arrayWith([
+            Match.objectLike({
+              Type: 'AWS::EC2::HostManagement',
+              Parameters: Match.arrayWith([
+                Match.objectLike({ Name: 'auto-release-host', Values: ['false'] })
+              ])
+            })
+          ])
+        });
+      });
+  });
+
+  it('non-dedicated-host runners keep maxCapacity equal to desired', () => {
+    runnerConfig.runnerTypes
+      .filter((type) => type.platform === PlatformType.AMAZONLINUX)
+      .forEach((type) => {
+        const stack = stacks.find((s) => s.stackName === generateASGStackName(type));
+        const template = Template.fromStack(stack!);
+        template.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', {
+          MaxSize: `${type.desiredInstances}`,
+          DesiredCapacity: `${type.desiredInstances}`
+        });
+      });
+  });
 });