fix(operator): schedule service wakers and tunnels on approved workers

changelog/unreleased/910-auxiliary-placement.md

@@ -0,0 +1,4 @@
+# Worker placement for wakers and relays
+
+Service-backed wakers and relay Pods can use an approved auxiliary worker pool. HostPort
+wakers retain the game’s placement and recorded endpoint node. [PR #910](https://github.com/GameplanePanel/Gameplane/pull/910)

charts/gameplane/templates/operator.yaml

@@ -412,6 +412,9 @@ spec:
             {{- if ne .Values.operator.gameWorkloadPolicy nil }}
             - {{ printf "--game-workload-policy=%s" (mustToJson .Values.operator.gameWorkloadPolicy) | quote }}
             {{- end }}
+            {{- with .Values.operator.auxiliaryWorkerPool }}
+            - {{ printf "--auxiliary-worker-pool=%s" . | quote }}
+            {{- end }}
             {{- if and .Values.operator.gameDataStorage .Values.operator.gameDataStorage.storageClassName }}
             - --game-data-storage-class={{ .Values.operator.gameDataStorage.storageClassName }}
             {{- end }}

charts/gameplane/values.yaml

@@ -88,6 +88,9 @@ operator:
   # Optional game worker label/NoSchedule taint and per-game tenant bounds.
   # See values-production.example.yaml and docs/production-placement.md.
   gameWorkloadPolicy: null
+  # Optional exact key=value label and matching NoSchedule taint for Service
+  # wakers and every tunnel provider. Defaults to the configured game pool.
+  auxiliaryWorkerPool: ""
   # ResourceQuota by managed game namespace. Each entry accepts hard and
   # an optional defaultContainerMemoryLimit for containers without a limit.
   tenantQuotas: {}

docs/production-placement.md

@@ -147,8 +147,35 @@ spec:
 
 Preserve control-plane taints. Avoid broad `operator: Exists` tolerations.
 Optional management components need their corresponding placement values too.
-Per-game sentinel and tunnel Deployments are operator-generated and are not
-covered by these chart management placement keys. A priorityClassName must name
+The operator places Service-backed wakers and FRP, playit and Tailscale relays
+on the game worker pool by default when `operator.gameWorkloadPolicy` is set.
+To run those components on a separate pool, set
+`operator.auxiliaryWorkerPool: gameplane.local/pool=relays`. Label the relay
+workers with that pair and taint them `gameplane.local/pool=relays:NoSchedule`.
+The operator adds the exact selector and toleration to each generated Deployment.
+A Hostport waker always uses the game placement and the recorded endpoint node;
+moving it to a relay worker would change the address players reconnect to.
+Relays and Service wakers do not mount the game PVC. A node-bound game PVC
+still determines where the game can resume after that relay receives a wake.
+
+If the pool is full, drained or absent, the scheduler leaves the listener
+Pending. Run `kubectl -n TENANT describe pod POD` and inspect its
+`PodScheduled=False` condition and `FailedScheduling` events for conflicting
+selectors, taints, affinity, or exhausted resources. Restore an eligible
+worker for the same pool (and the original endpoint for Hostport).
+
+The disposable kind management drill runs the operator on a tainted management
+worker and provisions local-path and topology-bound CSI game volumes on a
+separate game worker. It reads each world's saved marker after a wake through
+ClusterIP, NodePort, the MetalLB address and Hostport. It drains each pool
+in turn. The relay and games must report an unschedulable replacement while
+their own worker is cordoned. After the game worker returns, the drill records
+recovery time and verifies both PVC and PV identities and saved markers.
+Each tunnel provider uses a local health responder. The CSI lane uses the
+upstream distributed host-path reference driver. Qualify the site's external
+network path and production CSI driver on its deployment topology.
+
+A priorityClassName must name
 an existing PriorityClass. Unsatisfied hard selectors, taints or topology
 constraints correctly leave pods Pending; do not silently relax them.
 

operator/cmd/main.go

@@ -164,6 +164,7 @@ func main() {
 		backupHardSameNode               bool
 		agentResourcesJSON               string
 		gameWorkloadPolicyJSON           string
+		auxiliaryWorkerPool              string
 		sentinelResourcesJSON            string
 		tunnelResourcesJSON              string
 		backupResourcesJSON              string
@@ -173,6 +174,7 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election.")
 	flag.StringVar(&agentResourcesJSON, "agent-resources", "", "JSON ResourceRequirements for agents and config init containers; default requests 25m/64Mi, {} clears defaults.")
 	flag.StringVar(&gameWorkloadPolicyJSON, "game-workload-policy", "", "JSON game worker pool and per-game resource bounds. Unset disables policy enforcement.")
+	flag.StringVar(&auxiliaryWorkerPool, "auxiliary-worker-pool", "", "Exact key=value node label and matching NoSchedule taint for service wakers and relays. Defaults to the configured game worker pool.")
 	flag.StringVar(&sentinelResourcesJSON, "sentinel-resources", "", "JSON ResourceRequirements for sentinels; default requests 25m/32Mi, {} clears defaults.")
 	flag.StringVar(&tunnelResourcesJSON, "tunnel-resources", "", "JSON ResourceRequirements for tunnel relays; default requests 25m/64Mi, {} clears defaults.")
 	flag.StringVar(&backupResourcesJSON, "backup-resources", "", "JSON ResourceRequirements for restic backup/init, restore and cleanup workers; default backup/restore requests 100m/128Mi, {} clears defaults.")
@@ -320,6 +322,11 @@ func main() {
 		setupLog.Error(err, "invalid --game-workload-policy")
 		os.Exit(1)
 	}
+	auxiliaryPool, err := controller.ParseAuxiliaryWorkerPool(auxiliaryWorkerPool)
+	if err != nil {
+		setupLog.Error(err, "invalid --auxiliary-worker-pool")
+		os.Exit(1)
+	}
 	agentResources, err := controller.ParseAuxiliaryResources(agentResourcesJSON)
 	if err != nil {
 		setupLog.Error(err, "invalid --agent-resources")
@@ -506,6 +513,7 @@ func main() {
 		CaptureEnabled:              captureEnabled,
 		RestrictedWorkloads:         restrictedWorkloads,
 		GameWorkloadPolicy:          gameWorkloadPolicy,
+		AuxiliaryWorkerPool:         auxiliaryPool,
 		AgentTokenExpirationSeconds: agentTokenExpirationSeconds,
 		DiagnosticsNamespace:        diagnosticsNamespace,
 		CaptureSidecarImage:         captureSidecarImage,

operator/internal/controller/auxiliary_placement_test.go

@@ -0,0 +1,70 @@
+package controller
+
+import (
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1"
+)
+
+func TestAuxiliaryWorkerPlacement(t *testing.T) {
+	for _, raw := range []string{"bad key=value", "pool=", "pool", "pool=a,b=c"} {
+		if _, err := ParseAuxiliaryWorkerPool(raw); err == nil {
+			t.Errorf("accepted invalid pool %q", raw)
+		}
+	}
+	r, gs, tmpl := revisionFixture(t)
+	ctx := t.Context()
+	gs.Spec.NodeSelector = map[string]string{"rack": "west"}
+	r.GameWorkloadPolicy, _, _ = gamePolicyFixture(t)
+	for _, step := range []struct{ setting, want string }{
+		{"gameplane.local/pool=relays", "relays"},
+		{"", "games"},
+		{"", ""},
+	} {
+		var err error
+		r.AuxiliaryWorkerPool, err = ParseAuxiliaryWorkerPool(step.setting)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if step.want == "" {
+			r.GameWorkloadPolicy = nil
+		}
+		check := func(suffix string) {
+			t.Helper()
+			var dep appsv1.Deployment
+			key := client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + suffix}
+			if err := r.Get(ctx, key, &dep); err != nil {
+				t.Fatal(err)
+			}
+			pod := dep.Spec.Template.Spec
+			if step.want == "" {
+				if len(pod.Tolerations) != 0 || len(pod.NodeSelector) != 0 {
+					t.Fatalf("stale placement on %s: %v %v", key, pod.NodeSelector, pod.Tolerations)
+				}
+				return
+			}
+			want := corev1.Toleration{Key: "gameplane.local/pool", Operator: corev1.TolerationOpEqual, Value: step.want, Effect: corev1.TaintEffectNoSchedule}
+			if len(pod.NodeSelector) != 1 || pod.NodeSelector[want.Key] != step.want || len(pod.Tolerations) != 1 || pod.Tolerations[0] != want {
+				t.Fatalf("placement on %s: %v %v", key, pod.NodeSelector, pod.Tolerations)
+			}
+		}
+		if err := r.reconcileSentinel(ctx, gs, tmpl, true); err != nil {
+			t.Fatal(err)
+		}
+		check("-waker")
+		for _, provider := range []string{"frp", "tailscale", "playit"} {
+			gs.Spec.Networking.Tunnel = &gameplanev1alpha1.GameServerTunnel{
+				Enabled: true, Provider: provider, Frp: &gameplanev1alpha1.FrpTunnelSpec{},
+				Tailscale: &gameplanev1alpha1.TailscaleTunnelSpec{}, Playit: &gameplanev1alpha1.PlayitTunnelSpec{},
+			}
+			if err := r.reconcileTunnel(ctx, gs, tmpl, true); err != nil {
+				t.Fatal(err)
+			}
+			check("-tunnel")
+		}
+	}
+}

operator/internal/controller/game_workload_policy.go

@@ -107,7 +107,7 @@ func ParseGameWorkloadPolicy(raw string) (*GameWorkloadPolicy, error) {
 	} else if err != io.EOF {
 		return nil, fmt.Errorf("decode trailing game workload policy data: %w", err)
 	}
-	if len(validation.IsQualifiedName(policy.PoolLabel)) != 0 || policy.PoolValue == "" || len(validation.IsValidLabelValue(policy.PoolValue)) != 0 {
+	if !validWorkerPool(policy.PoolLabel, policy.PoolValue) {
 		return nil, fmt.Errorf("poolLabel and poolValue must identify a valid, nonempty worker label")
 	}
 	if policy.MinCPURequest.Sign() <= 0 || policy.MaxCPURequest.Cmp(policy.MinCPURequest) < 0 ||
@@ -153,10 +153,43 @@ func (p *GameWorkloadPolicy) placement(gs *gameplanev1alpha1.GameServer) (map[st
 	if p == nil {
 		return gs.Spec.NodeSelector, gs.Spec.Tolerations
 	}
-	selector := map[string]string{}
-	maps.Copy(selector, gs.Spec.NodeSelector)
-	selector[p.PoolLabel] = p.PoolValue
+	return workerPlacement(p.PoolLabel, p.PoolValue, gs.Spec.NodeSelector)
+}
+
+func validWorkerPool(key, value string) bool {
+	return len(validation.IsQualifiedName(key)) == 0 && value != "" && len(validation.IsValidLabelValue(value)) == 0
+}
+
+func workerPlacement(key, value string, extra map[string]string) (map[string]string, []corev1.Toleration) {
+	selector := make(map[string]string, len(extra)+1)
+	maps.Copy(selector, extra)
+	selector[key] = value
 	return selector, []corev1.Toleration{{
-		Key: p.PoolLabel, Operator: corev1.TolerationOpEqual, Value: p.PoolValue, Effect: corev1.TaintEffectNoSchedule,
+		Key: key, Operator: corev1.TolerationOpEqual, Value: value, Effect: corev1.TaintEffectNoSchedule,
 	}}
 }
+
+// AuxiliaryWorkerPool selects the worker pool for Service wakers and relays.
+type AuxiliaryWorkerPool struct{ key, value string }
+
+// ParseAuxiliaryWorkerPool accepts one exact label and matching NoSchedule taint.
+func ParseAuxiliaryWorkerPool(raw string) (*AuxiliaryWorkerPool, error) {
+	if raw == "" {
+		return nil, nil
+	}
+	key, value, ok := strings.Cut(raw, "=")
+	if !ok || !validWorkerPool(key, value) {
+		return nil, fmt.Errorf("expected one worker label key=value, got %q", raw)
+	}
+	return &AuxiliaryWorkerPool{key: key, value: value}, nil
+}
+
+func (r *GameServerReconciler) auxiliaryPlacement() (map[string]string, []corev1.Toleration) {
+	if p := r.AuxiliaryWorkerPool; p != nil {
+		return workerPlacement(p.key, p.value, nil)
+	}
+	if p := r.GameWorkloadPolicy; p != nil {
+		return workerPlacement(p.PoolLabel, p.PoolValue, nil)
+	}
+	return nil, nil
+}

operator/internal/controller/gameserver_controller.go

@@ -180,6 +180,7 @@ type GameServerReconciler struct {
 	// RestrictedWorkloads applies the opt-in PSA-compatible game profile.
 	RestrictedWorkloads         bool
 	GameWorkloadPolicy          *GameWorkloadPolicy
+	AuxiliaryWorkerPool         *AuxiliaryWorkerPool
 	AgentTokenExpirationSeconds int64
 	DiagnosticsNamespace        string
 

operator/internal/controller/gameserver_sentinel.go

@@ -337,13 +337,11 @@ func (r *GameServerReconciler) reconcileSentinel(
 
 		dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs)
 		if hostPort {
-			dep.Spec.Template.Spec.NodeSelector = gs.Spec.NodeSelector
-			dep.Spec.Template.Spec.Tolerations = gs.Spec.Tolerations
+			dep.Spec.Template.Spec.NodeSelector, dep.Spec.Template.Spec.Tolerations = r.GameWorkloadPolicy.placement(gs)
 			dep.Spec.Template.Spec.Affinity = hostportAffinity(gs)
 			dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}
 		} else {
-			dep.Spec.Template.Spec.NodeSelector = nil
-			dep.Spec.Template.Spec.Tolerations = nil
+			dep.Spec.Template.Spec.NodeSelector, dep.Spec.Template.Spec.Tolerations = r.auxiliaryPlacement()
 			dep.Spec.Template.Spec.Affinity = nil
 		}
 		dep.Spec.Template.Spec.Containers = []corev1.Container{{

operator/internal/controller/gameserver_sentinel_envtest_test.go

@@ -182,7 +182,8 @@ func TestSentinelReconciliation(t *testing.T) {
 		}
 		deleteCleanup(t, tmpl)
 
-		node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("sentinel-node")}}
+		policy, _, _ := gamePolicyFixture(t)
+		node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("sentinel-node"), Labels: map[string]string{policy.PoolLabel: policy.PoolValue}}}
 		if err := k8sClient.Create(ctx, node); err != nil {
 			t.Fatalf("create node: %v", err)
 		}
@@ -206,7 +207,8 @@ func TestSentinelReconciliation(t *testing.T) {
 			t.Fatalf("create gameserver: %v", err)
 		}
 
-		r := &GameServerReconciler{Client: k8sClient, APIReader: k8sClient, Scheme: scheme}
+		r := &GameServerReconciler{Client: k8sClient, APIReader: k8sClient, Scheme: scheme,
+			GameWorkloadPolicy: policy, AuxiliaryWorkerPool: &AuxiliaryWorkerPool{key: policy.PoolLabel, value: "relays"}}
 		if err := r.reconcileSentinel(ctx, gs, tmpl, true); err != nil {
 			t.Fatalf("reconcileSentinel: %v", err)
 		}
@@ -217,6 +219,11 @@ func TestSentinelReconciliation(t *testing.T) {
 		}, &dep); err != nil {
 			t.Fatalf("get sentinel deployment: %v", err)
 		}
+		pod := dep.Spec.Template.Spec
+		if pod.NodeSelector[policy.PoolLabel] != policy.PoolValue || len(pod.Tolerations) != 1 || pod.Tolerations[0].Value != policy.PoolValue ||
+			pod.Affinity == nil || pod.Affinity.NodeAffinity == nil {
+			t.Fatalf("hostport listener lost game pool or endpoint affinity: %+v", pod)
+		}
 		if len(dep.Spec.Template.Spec.Containers) == 0 {
 			t.Fatal("no containers in sentinel deployment")
 		}

operator/internal/controller/gameserver_tunnel.go

@@ -229,6 +229,7 @@ func (r *GameServerReconciler) reconcileTunnel(
 		refusedSecret = ""
 		replicas := int32(1)
 		dep.Spec.Replicas = &replicas
+		dep.Spec.Template.Spec.NodeSelector, dep.Spec.Template.Spec.Tolerations = r.auxiliaryPlacement()
 		// A provider identity owns one relay session. Recreate unregisters the
 		// old child before its replacement can claim the same names/ports.
 		if tunnel.Provider == "frp" || tunnel.Provider == "playit" || tunnel.Provider == "tailscale" {