From c462731675a90ab9492f13f70a2797479b812a7c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 00:19:39 -0500 Subject: [PATCH 1/4] fix(operator): place wakers and relays on the approved worker pool Signed-off-by: Baylem --- charts/gameplane/templates/operator.yaml | 3 + charts/gameplane/values.yaml | 3 + docs/production-placement.md | 19 ++++- operator/cmd/main.go | 8 +++ .../controller/auxiliary_placement_test.go | 70 +++++++++++++++++++ .../controller/game_workload_policy.go | 43 ++++++++++-- .../controller/gameserver_controller.go | 1 + .../controller/gameserver_sentinel.go | 6 +- .../gameserver_sentinel_envtest_test.go | 11 ++- .../internal/controller/gameserver_tunnel.go | 1 + 10 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 operator/internal/controller/auxiliary_placement_test.go diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index c2995d351..6d4f6e767 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -412,6 +412,9 @@ spec: {{- if ne .Values.operator.gameWorkloadPolicy nil }} - {{ printf "--game-workload-policy=%s" (mustToJson .Values.operator.gameWorkloadPolicy) | quote }} {{- end }} + {{- with .Values.operator.auxiliaryWorkerPool }} + - {{ printf "--auxiliary-worker-pool=%s" . | quote }} + {{- end }} {{- if and .Values.operator.gameDataStorage .Values.operator.gameDataStorage.storageClassName }} - --game-data-storage-class={{ .Values.operator.gameDataStorage.storageClassName }} {{- end }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 42fffe9dd..736207603 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -88,6 +88,9 @@ operator: # Optional game worker label/NoSchedule taint and per-game tenant bounds. # See values-production.example.yaml and docs/production-placement.md. gameWorkloadPolicy: null + # Optional exact key=value label and matching NoSchedule taint for Service + # wakers and every tunnel provider. Defaults to the configured game pool. + auxiliaryWorkerPool: "" # ResourceQuota by managed game namespace. Each entry accepts hard and # an optional defaultContainerMemoryLimit for containers without a limit. tenantQuotas: {} diff --git a/docs/production-placement.md b/docs/production-placement.md index b6a6963d4..355f93b4f 100644 --- a/docs/production-placement.md +++ b/docs/production-placement.md @@ -143,8 +143,23 @@ spec: Preserve control-plane taints. Avoid broad `operator: Exists` tolerations. Optional management components need their corresponding placement values too. -Per-game sentinel and tunnel Deployments are operator-generated and are not -covered by these chart management placement keys. A priorityClassName must name +The operator places Service-backed wakers and FRP, playit and Tailscale relays +on the game worker pool by default when `operator.gameWorkloadPolicy` is set. +To run those components on a separate pool, set +`operator.auxiliaryWorkerPool: gameplane.local/pool=relays`. Label the relay +workers with that pair and taint them `gameplane.local/pool=relays:NoSchedule`. +The operator adds the exact selector and toleration to each generated Deployment. +A Hostport waker always uses the game placement and the recorded endpoint node; +moving it to a relay worker would change the address players reconnect to. +Relays and Service wakers do not mount the game PVC. A node-bound game PVC +still determines where the game can resume after that relay receives a wake. + +If the pool is full, drained or absent, the scheduler leaves the listener +Pending. Run `kubectl -n TENANT describe pod POD` and inspect its +`PodScheduled=False` condition and `FailedScheduling` events for conflicting +selectors, taints, affinity, or exhausted resources. Restore an eligible +worker for the same pool (and the original endpoint for Hostport). +A priorityClassName must name an existing PriorityClass. Unsatisfied hard selectors, taints or topology constraints correctly leave pods Pending; do not silently relax them. diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 4546777c8..a5bb0b1f1 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -164,6 +164,7 @@ func main() { backupHardSameNode bool agentResourcesJSON string gameWorkloadPolicyJSON string + auxiliaryWorkerPool string sentinelResourcesJSON string tunnelResourcesJSON string backupResourcesJSON string @@ -173,6 +174,7 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election.") flag.StringVar(&agentResourcesJSON, "agent-resources", "", "JSON ResourceRequirements for agents and config init containers; default requests 25m/64Mi, {} clears defaults.") flag.StringVar(&gameWorkloadPolicyJSON, "game-workload-policy", "", "JSON game worker pool and per-game resource bounds. Unset disables policy enforcement.") + flag.StringVar(&auxiliaryWorkerPool, "auxiliary-worker-pool", "", "Exact key=value node label and matching NoSchedule taint for service wakers and relays. Defaults to the configured game worker pool.") flag.StringVar(&sentinelResourcesJSON, "sentinel-resources", "", "JSON ResourceRequirements for sentinels; default requests 25m/32Mi, {} clears defaults.") flag.StringVar(&tunnelResourcesJSON, "tunnel-resources", "", "JSON ResourceRequirements for tunnel relays; default requests 25m/64Mi, {} clears defaults.") flag.StringVar(&backupResourcesJSON, "backup-resources", "", "JSON ResourceRequirements for restic backup/init, restore and cleanup workers; default backup/restore requests 100m/128Mi, {} clears defaults.") @@ -320,6 +322,11 @@ func main() { setupLog.Error(err, "invalid --game-workload-policy") os.Exit(1) } + auxiliaryPool, err := controller.ParseAuxiliaryWorkerPool(auxiliaryWorkerPool) + if err != nil { + setupLog.Error(err, "invalid --auxiliary-worker-pool") + os.Exit(1) + } agentResources, err := controller.ParseAuxiliaryResources(agentResourcesJSON) if err != nil { setupLog.Error(err, "invalid --agent-resources") @@ -506,6 +513,7 @@ func main() { CaptureEnabled: captureEnabled, RestrictedWorkloads: restrictedWorkloads, GameWorkloadPolicy: gameWorkloadPolicy, + AuxiliaryWorkerPool: auxiliaryPool, AgentTokenExpirationSeconds: agentTokenExpirationSeconds, DiagnosticsNamespace: diagnosticsNamespace, CaptureSidecarImage: captureSidecarImage, diff --git a/operator/internal/controller/auxiliary_placement_test.go b/operator/internal/controller/auxiliary_placement_test.go new file mode 100644 index 000000000..81525b2f6 --- /dev/null +++ b/operator/internal/controller/auxiliary_placement_test.go @@ -0,0 +1,70 @@ +package controller + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestAuxiliaryWorkerPlacement(t *testing.T) { + for _, raw := range []string{"bad key=value", "pool=", "pool", "pool=a,b=c"} { + if _, err := ParseAuxiliaryWorkerPool(raw); err == nil { + t.Errorf("accepted invalid pool %q", raw) + } + } + r, gs, tmpl := revisionFixture(t) + ctx := t.Context() + gs.Spec.NodeSelector = map[string]string{"rack": "west"} + r.GameWorkloadPolicy, _, _ = gamePolicyFixture(t) + for _, step := range []struct{ setting, want string }{ + {"gameplane.local/pool=relays", "relays"}, + {"", "games"}, + {"", ""}, + } { + var err error + r.AuxiliaryWorkerPool, err = ParseAuxiliaryWorkerPool(step.setting) + if err != nil { + t.Fatal(err) + } + if step.want == "" { + r.GameWorkloadPolicy = nil + } + check := func(suffix string) { + t.Helper() + var dep appsv1.Deployment + key := client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + suffix} + if err := r.Get(ctx, key, &dep); err != nil { + t.Fatal(err) + } + pod := dep.Spec.Template.Spec + if step.want == "" { + if len(pod.Tolerations) != 0 || len(pod.NodeSelector) != 0 { + t.Fatalf("stale placement on %s: %v %v", key, pod.NodeSelector, pod.Tolerations) + } + return + } + want := corev1.Toleration{Key: "gameplane.local/pool", Operator: corev1.TolerationOpEqual, Value: step.want, Effect: corev1.TaintEffectNoSchedule} + if len(pod.NodeSelector) != 1 || pod.NodeSelector[want.Key] != step.want || len(pod.Tolerations) != 1 || pod.Tolerations[0] != want { + t.Fatalf("placement on %s: %v %v", key, pod.NodeSelector, pod.Tolerations) + } + } + if err := r.reconcileSentinel(ctx, gs, tmpl, true); err != nil { + t.Fatal(err) + } + check("-waker") + for _, provider := range []string{"frp", "tailscale", "playit"} { + gs.Spec.Networking.Tunnel = &gameplanev1alpha1.GameServerTunnel{ + Enabled: true, Provider: provider, Frp: &gameplanev1alpha1.FrpTunnelSpec{}, + Tailscale: &gameplanev1alpha1.TailscaleTunnelSpec{}, Playit: &gameplanev1alpha1.PlayitTunnelSpec{}, + } + if err := r.reconcileTunnel(ctx, gs, tmpl, true); err != nil { + t.Fatal(err) + } + check("-tunnel") + } + } +} diff --git a/operator/internal/controller/game_workload_policy.go b/operator/internal/controller/game_workload_policy.go index 5db30913f..86fc2dde3 100644 --- a/operator/internal/controller/game_workload_policy.go +++ b/operator/internal/controller/game_workload_policy.go @@ -39,7 +39,7 @@ func ParseGameWorkloadPolicy(raw string) (*GameWorkloadPolicy, error) { if decoder.Decode(new(any)) != io.EOF { return nil, fmt.Errorf("expected a single game workload policy object") } - if len(validation.IsQualifiedName(policy.PoolLabel)) != 0 || policy.PoolValue == "" || len(validation.IsValidLabelValue(policy.PoolValue)) != 0 { + if !validWorkerPool(policy.PoolLabel, policy.PoolValue) { return nil, fmt.Errorf("poolLabel and poolValue must identify a valid, nonempty worker label") } if policy.MinCPURequest.Sign() <= 0 || policy.MaxCPURequest.Cmp(policy.MinCPURequest) < 0 || @@ -83,10 +83,43 @@ func (p *GameWorkloadPolicy) placement(gs *gameplanev1alpha1.GameServer) (map[st if p == nil { return gs.Spec.NodeSelector, gs.Spec.Tolerations } - selector := map[string]string{} - maps.Copy(selector, gs.Spec.NodeSelector) - selector[p.PoolLabel] = p.PoolValue + return workerPlacement(p.PoolLabel, p.PoolValue, gs.Spec.NodeSelector) +} + +func validWorkerPool(key, value string) bool { + return len(validation.IsQualifiedName(key)) == 0 && value != "" && len(validation.IsValidLabelValue(value)) == 0 +} + +func workerPlacement(key, value string, extra map[string]string) (map[string]string, []corev1.Toleration) { + selector := make(map[string]string, len(extra)+1) + maps.Copy(selector, extra) + selector[key] = value return selector, []corev1.Toleration{{ - Key: p.PoolLabel, Operator: corev1.TolerationOpEqual, Value: p.PoolValue, Effect: corev1.TaintEffectNoSchedule, + Key: key, Operator: corev1.TolerationOpEqual, Value: value, Effect: corev1.TaintEffectNoSchedule, }} } + +// AuxiliaryWorkerPool selects the worker pool for Service wakers and relays. +type AuxiliaryWorkerPool struct{ key, value string } + +// ParseAuxiliaryWorkerPool accepts one exact label and matching NoSchedule taint. +func ParseAuxiliaryWorkerPool(raw string) (*AuxiliaryWorkerPool, error) { + if raw == "" { + return nil, nil + } + key, value, ok := strings.Cut(raw, "=") + if !ok || !validWorkerPool(key, value) { + return nil, fmt.Errorf("expected one worker label key=value, got %q", raw) + } + return &AuxiliaryWorkerPool{key: key, value: value}, nil +} + +func (r *GameServerReconciler) auxiliaryPlacement() (map[string]string, []corev1.Toleration) { + if p := r.AuxiliaryWorkerPool; p != nil { + return workerPlacement(p.key, p.value, nil) + } + if p := r.GameWorkloadPolicy; p != nil { + return workerPlacement(p.PoolLabel, p.PoolValue, nil) + } + return nil, nil +} diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index a8c849702..e33dd6a64 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -180,6 +180,7 @@ type GameServerReconciler struct { // RestrictedWorkloads applies the opt-in PSA-compatible game profile. RestrictedWorkloads bool GameWorkloadPolicy *GameWorkloadPolicy + AuxiliaryWorkerPool *AuxiliaryWorkerPool AgentTokenExpirationSeconds int64 DiagnosticsNamespace string diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index cb263fe7e..dd0650f4c 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -337,13 +337,11 @@ func (r *GameServerReconciler) reconcileSentinel( dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs) if hostPort { - dep.Spec.Template.Spec.NodeSelector = gs.Spec.NodeSelector - dep.Spec.Template.Spec.Tolerations = gs.Spec.Tolerations + dep.Spec.Template.Spec.NodeSelector, dep.Spec.Template.Spec.Tolerations = r.GameWorkloadPolicy.placement(gs) dep.Spec.Template.Spec.Affinity = hostportAffinity(gs) dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} } else { - dep.Spec.Template.Spec.NodeSelector = nil - dep.Spec.Template.Spec.Tolerations = nil + dep.Spec.Template.Spec.NodeSelector, dep.Spec.Template.Spec.Tolerations = r.auxiliaryPlacement() dep.Spec.Template.Spec.Affinity = nil } dep.Spec.Template.Spec.Containers = []corev1.Container{{ diff --git a/operator/internal/controller/gameserver_sentinel_envtest_test.go b/operator/internal/controller/gameserver_sentinel_envtest_test.go index bf79d3c8a..f8e1d9d60 100644 --- a/operator/internal/controller/gameserver_sentinel_envtest_test.go +++ b/operator/internal/controller/gameserver_sentinel_envtest_test.go @@ -182,7 +182,8 @@ func TestSentinelReconciliation(t *testing.T) { } deleteCleanup(t, tmpl) - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("sentinel-node")}} + policy, _, _ := gamePolicyFixture(t) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("sentinel-node"), Labels: map[string]string{policy.PoolLabel: policy.PoolValue}}} if err := k8sClient.Create(ctx, node); err != nil { t.Fatalf("create node: %v", err) } @@ -206,7 +207,8 @@ func TestSentinelReconciliation(t *testing.T) { t.Fatalf("create gameserver: %v", err) } - r := &GameServerReconciler{Client: k8sClient, APIReader: k8sClient, Scheme: scheme} + r := &GameServerReconciler{Client: k8sClient, APIReader: k8sClient, Scheme: scheme, + GameWorkloadPolicy: policy, AuxiliaryWorkerPool: &AuxiliaryWorkerPool{key: policy.PoolLabel, value: "relays"}} if err := r.reconcileSentinel(ctx, gs, tmpl, true); err != nil { t.Fatalf("reconcileSentinel: %v", err) } @@ -217,6 +219,11 @@ func TestSentinelReconciliation(t *testing.T) { }, &dep); err != nil { t.Fatalf("get sentinel deployment: %v", err) } + pod := dep.Spec.Template.Spec + if pod.NodeSelector[policy.PoolLabel] != policy.PoolValue || len(pod.Tolerations) != 1 || pod.Tolerations[0].Value != policy.PoolValue || + pod.Affinity == nil || pod.Affinity.NodeAffinity == nil { + t.Fatalf("hostport listener lost game pool or endpoint affinity: %+v", pod) + } if len(dep.Spec.Template.Spec.Containers) == 0 { t.Fatal("no containers in sentinel deployment") } diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index 7764240c8..1be828c87 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -229,6 +229,7 @@ func (r *GameServerReconciler) reconcileTunnel( refusedSecret = "" replicas := int32(1) dep.Spec.Replicas = &replicas + dep.Spec.Template.Spec.NodeSelector, dep.Spec.Template.Spec.Tolerations = r.auxiliaryPlacement() // A provider identity owns one relay session. Recreate unregisters the // old child before its replacement can claim the same names/ports. if tunnel.Provider == "frp" || tunnel.Provider == "playit" || tunnel.Provider == "tailscale" { From 7a2e26cef7c6cd629638a377707cdaf87b6d5f60 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 00:35:33 -0500 Subject: [PATCH 2/4] test(e2e): exercise auxiliary placement through drain and wake handoff Signed-off-by: Baylem --- docs/production-placement.md | 10 ++ test/e2e/auxiliary-placement.sh | 159 +++++++++++++++++++++++ test/e2e/auxiliary_placement_e2e_test.go | 21 +++ test/e2e/buckets.sh | 1 + test/e2e/suite-components.json | 1 + 5 files changed, 192 insertions(+) create mode 100644 test/e2e/auxiliary-placement.sh create mode 100644 test/e2e/auxiliary_placement_e2e_test.go diff --git a/docs/production-placement.md b/docs/production-placement.md index 5640a2011..670f6e33e 100644 --- a/docs/production-placement.md +++ b/docs/production-placement.md @@ -159,6 +159,16 @@ Pending. Run `kubectl -n TENANT describe pod POD` and inspect its `PodScheduled=False` condition and `FailedScheduling` events for conflicting selectors, taints, affinity, or exhausted resources. Restore an eligible worker for the same pool (and the original endpoint for Hostport). + +The disposable kind management drill runs the operator on a tainted management +worker, binds a populated local-path PVC on a separate game worker, parks +the game and wakes it over all four exposure modes. It reads the original +world marker after each handoff. It also verifies the scheduler's diagnostic +after evicting a relay from its only eligible worker. Each provider uses a +local health responder so this placement drill needs no external accounts. +The LoadBalancer case exercises its allocated NodePort. Test a real external +address and the site's CSI driver on the deployment topology before rollout. + A priorityClassName must name an existing PriorityClass. Unsatisfied hard selectors, taints or topology constraints correctly leave pods Pending; do not silently relax them. diff --git a/test/e2e/auxiliary-placement.sh b/test/e2e/auxiliary-placement.sh new file mode 100644 index 000000000..ee7bd6445 --- /dev/null +++ b/test/e2e/auxiliary-placement.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# The management bucket owns this disposable kind cluster and its operator. +set -euo pipefail +cluster="${GAMEPLANE_E2E_CLUSTER:?management kind cluster required}" +context="kind-${cluster}" +[[ "$(kubectl config current-context)" == "$context" ]] +k() { kubectl --context="$context" "$@"; } +ns=gameplane-games +panel=gameplane-system +game=aux-worker-drill +key=gameplane.local/aux-drill +image=gameplane-test/placement-relay:ci +op=deployment/gameplane-operator +work=$(mktemp -d) +mapfile -t workers < <(k get nodes -o json | jq -r '.items[] | select(.metadata.labels["node-role.kubernetes.io/control-plane"] == null and (.spec.unschedulable != true)) | .metadata.name') +[[ ${#workers[@]} == 2 ]] +games=${workers[0]} +aux=${workers[1]} +k get nodes -l node-role.kubernetes.io/control-plane -o json | jq -e 'any(.items[].spec.taints[]; .key=="node-role.kubernetes.io/control-plane" and .effect=="NoSchedule")' >/dev/null +k get "$op" -n "$panel" -o json | jq '[{op:"replace",path:"/spec/template",value:.spec.template}]' > "$work/restore.json" +cleanup() { + k uncordon "$aux" >/dev/null || true + for node in "$games" "$aux"; do + k taint node "$node" "$key:NoSchedule-" >/dev/null || true + k label node "$node" "$key-" >/dev/null || true + done + k delete gameserver "$game" -n "$ns" --ignore-not-found --wait=false + k delete gametemplate "$game" --ignore-not-found --wait=false + k patch "$op" -n "$panel" --type=json --patch-file "$work/restore.json" >/dev/null + k rollout status "$op" -n "$panel" --timeout=3m +} +trap cleanup EXIT +wait_for() { + local until_time=$((SECONDS + 150)) + until "$@"; do + if (( SECONDS >= until_time )); then + k get pods -n "$ns" -o wide + k get events -n "$ns" --sort-by=.lastTimestamp | tail -25 + echo "timed out waiting for $*" >&2 + return 1 + fi + sleep 2 + done +} +# A local HTTP responder supplies each provider's health contract without +# registering test identities with public relay providers. +cat > "$work/Dockerfile" <<'EOF' +FROM busybox:1.36 +ENTRYPOINT ["sh","-ec","mkdir -p /tmp/site; echo ok >/tmp/site/healthz; echo ready >/tmp/site/readyz; exec httpd -f -p 8091 -h /tmp/site"] +EOF +docker build --quiet -t "$image" "$work" +kind load docker-image --name "$cluster" "$image" +k label node "$games" "$key=games" +k label node "$aux" "$key=management" +# Provision the node-bound game claim before adding any taint to the worker. +# kind's local-path storage helper does not tolerate arbitrary custom taints. +k apply -f - </data/world; exec httpd -f -p 12345 -h /data'] + storage: {size: 64Mi, mountPath: /data} + resources: {requests: {cpu: 10m, memory: 8Mi}, limits: {memory: 32Mi}} + ports: [{name: game, containerPort: 12345, advertise: true, protocol: TCP, wakeProtocol: generic}] +--- +apiVersion: gameplane.local/v1alpha1 +kind: GameServer +metadata: {name: $game, namespace: $ns} +spec: + templateRef: {name: $game} + nodeSelector: {$key: games} + idle: {enabled: true, afterMinutes: 60, wakeOnConnect: true} +EOF +world() { k -n "$ns" exec "$game-0" -c game -- cat /data/world; } +has_world() { [[ -n "$(world 2>/dev/null)" ]]; } +wait_for has_world +marker=$(world) +claim=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}') +pv=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.spec.volumeName}') +k get pv "$pv" -o json | jq -e '.spec.nodeAffinity.required.nodeSelectorTerms | length > 0' >/dev/null +policy=$(jq -nc --arg key "$key" '{poolLabel:$key,poolValue:"games",minCPURequest:"10m",maxCPURequest:"1",minMemoryRequest:"8Mi",maxMemoryLimit:"32Mi"}') +jq -nc --arg key "$key" '{spec:{template:{spec:{nodeSelector:{($key):"management"},tolerations:[{key:$key,operator:"Equal",value:"management",effect:"NoSchedule"}]}}}}' > "$work/pool.json" +k -n "$panel" patch "$op" --type=merge --patch-file "$work/pool.json" +args=$(k -n "$panel" get "$op" -o json | jq -c --arg p "$policy" --arg key "$key" --arg image "$image" ' + .spec.template.spec.containers[0].args | map(select(test("^--(game-workload-policy|auxiliary-worker-pool|tunnel-(frp|tailscale|playit)-image)=") | not)) + + ["--game-workload-policy="+$p,"--auxiliary-worker-pool="+$key+"=management","--tunnel-frp-image="+$image,"--tunnel-tailscale-image="+$image,"--tunnel-playit-image="+$image] | + [{op:"replace",path:"/spec/template/spec/containers/0/args",value:.}]') +k -n "$panel" patch "$op" --type=json -p "$args" +k rollout status "$op" -n "$panel" --timeout=3m +for pool in games management; do + node=$games; [[ "$pool" != management ]] || node=$aux + k taint node "$node" "$key=$pool:NoSchedule" +done +gsuid=$(k -n "$ns" get gameserver "$game" -o jsonpath='{.metadata.uid}') +k -n "$ns" create secret generic "$game" --from-literal=token=fixture --from-literal=authKey=fixture --from-literal=secretKey=fixture --dry-run=client -o json | + jq --arg game "$game" --arg uid "$gsuid" '.metadata.ownerReferences=[{apiVersion:"gameplane.local/v1alpha1",kind:"GameServer",name:$game,uid:$uid,controller:true}]' | k apply -f - +patch_game() { k -n "$ns" patch gameserver "$game" --type=merge -p "$1" >/dev/null; } +pod_on() { + k -n "$ns" get pods -l "app.kubernetes.io/instance=$game,app.kubernetes.io/name=gameplane-$1" -o json | + jq -er --arg node "$2" '.items[] | select(.metadata.deletionTimestamp==null and .spec.nodeName==$node and any(.status.conditions[]?;.type=="Ready" and .status=="True")) | .metadata.name' | head -1 +} +for provider in frp tailscale playit; do + cfg=$(jq -nc --arg p "$provider" --arg cred "$game" '{enabled:true,provider:$p,credentialsSecretRef:{name:$cred},frp:null,tailscale:null,playit:null} | + .[$p]=(if $p=="frp" then {serverAddr:"127.0.0.1",remotePorts:[{name:"game",remotePort:12345}]} else {} end) | {spec:{networking:{tunnel:.}}}') + patch_game "$cfg" + provider_ready() { + k -n "$ns" get deployment "$game-tunnel" -o json | jq -e --arg p "$provider" 'any(.spec.template.spec.containers[0].env[]; .name=="TUNNEL_TYPE" and .value==$p)' >/dev/null && + k -n "$ns" rollout status "deployment/$game-tunnel" --timeout=8s >/dev/null && pod_on tunnel "$aux" >/dev/null + } + wait_for provider_ready + echo "$provider relay ready on $aux" +done +relay=$(pod_on tunnel "$aux") +k cordon "$aux" +k -n "$ns" delete pod "$relay" --wait=false +unschedulable() { + k -n "$ns" get pods -l "app.kubernetes.io/name=gameplane-tunnel,app.kubernetes.io/instance=$game" -o json | + jq -e 'any(.items[]; .metadata.deletionTimestamp==null and any(.status.conditions[]?; .type=="PodScheduled" and .reason=="Unschedulable"))' >/dev/null +} +wait_for unschedulable +k uncordon "$aux" +wait_for pod_on tunnel "$aux" +relay=$(pod_on tunnel "$aux") +nodeIP=$(k get node "$games" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') +for mode in ClusterIP NodePort LoadBalancer Hostport; do + patch_game "$(jq -nc --arg mode "$mode" '{spec:{networking:{expose:$mode}}}')" + # Wait for this exposure's desired game before parking it. + mode_ready() { + service_type=$mode; [[ "$mode" != Hostport ]] || service_type=ClusterIP + [[ "$(k -n "$ns" get service "$game" -o jsonpath='{.spec.type}')" == "$service_type" ]] && + k -n "$ns" get sts "$game" -o json | jq -e --arg mode "$mode" '(.status.readyReplicas==1) and + (([.spec.template.spec.containers[]|select(.name=="game")|.ports[]|.hostPort // 0]|max > 0)==($mode=="Hostport"))' >/dev/null + } + wait_for mode_ready + k -n "$ns" annotate gameserver "$game" "gameplane.local/idle-asleep-since=$(date -u +%FT%TZ)" --overwrite + endpoint=$game.$ns.svc:12345; worker=$aux + if [[ "$mode" == Hostport ]]; then + worker=$games + endpoint=$nodeIP:12345 + elif [[ "$mode" != ClusterIP ]]; then + endpoint=$nodeIP:$(k -n "$ns" get service "$game" -o jsonpath='{.spec.ports[0].nodePort}') + fi + wait_for pod_on waker "$worker" + parked() { + [[ "$(k -n "$ns" get sts "$game" -o jsonpath='{.spec.replicas}')" == 0 && + -z "$(k -n "$ns" get pod "$game-0" --ignore-not-found -o name)" ]] + } + wait_for parked + join_world() { [[ "$(k -n "$ns" exec "$relay" -- wget -T 6 -qO- "http://$endpoint/world" 2>/dev/null)" == "$marker" ]]; } + wait_for join_world + [[ "$(k -n "$ns" get pod "$game-0" -o jsonpath='{.spec.nodeName}')" == "$games" ]] + [[ "$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}')" == "$claim" ]] + echo "$mode waker=$worker relay=$aux game=$games preserved PVC=$claim world=$marker" +done diff --git a/test/e2e/auxiliary_placement_e2e_test.go b/test/e2e/auxiliary_placement_e2e_test.go new file mode 100644 index 000000000..e93367592 --- /dev/null +++ b/test/e2e/auxiliary_placement_e2e_test.go @@ -0,0 +1,21 @@ +//go:build e2e + +package e2e + +import ( + "os/exec" + "testing" +) + +func TestAuxiliaryWorkersAndWakeHandoff(t *testing.T) { + path, err := findRepoRel("test/e2e/auxiliary-placement.sh") + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(t.Context(), "bash", path) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("auxiliary worker drill: %v\n%s", err, out) + } else { + t.Logf("%s", out) + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index 4d3a32bf0..aaa36ecd7 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -442,6 +442,7 @@ EOF bucket_management_placement() { cat <<'EOF' TestManagementPlacementAndDisruption TestGameWorkerPoolAndQuota +TestAuxiliaryWorkersAndWakeHandoff EOF } diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index c4fc7c3a5..f0567e2b9 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -25,6 +25,7 @@ { "tests": "^TestRestrictedWorkloads_", "components": ["operator", "api", "agent", "capture-sidecar"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, { "tests": "^TestGameWorkerPoolAndQuota$", "components": ["operator"] }, + { "tests": "^TestAuxiliaryWorkersAndWakeHandoff$", "components": ["operator", "sentinel", "tunnel"], "paths": ["test/e2e/auxiliary-placement.sh"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] }, { "tests": "^TestGameServer_", "components": ["operator", "api", "agent"] }, From e6df3b8199267b0e8b7d2a3cf42d1fd491f86239 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 01:08:37 -0500 Subject: [PATCH 3/4] test(operator): qualify pinned CSI volumes through wake handoffs Signed-off-by: Baylem --- test/e2e/auxiliary-placement.sh | 120 ++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 38 deletions(-) diff --git a/test/e2e/auxiliary-placement.sh b/test/e2e/auxiliary-placement.sh index ee7bd6445..4a93ec1d3 100644 --- a/test/e2e/auxiliary-placement.sh +++ b/test/e2e/auxiliary-placement.sh @@ -3,14 +3,18 @@ set -euo pipefail cluster="${GAMEPLANE_E2E_CLUSTER:?management kind cluster required}" context="kind-${cluster}" +[[ "$cluster" == gameplane-e2e ]] [[ "$(kubectl config current-context)" == "$context" ]] k() { kubectl --context="$context" "$@"; } ns=gameplane-games panel=gameplane-system game=aux-worker-drill +local_game=$game +csi_game=${game}-csi key=gameplane.local/aux-drill image=gameplane-test/placement-relay:ci op=deployment/gameplane-operator +[[ "$(k -n "$panel" get "$op" -o jsonpath='{.spec.template.spec.containers[0].image}')" == gameplane-test/operator:* ]] work=$(mktemp -d) mapfile -t workers < <(k get nodes -o json | jq -r '.items[] | select(.metadata.labels["node-role.kubernetes.io/control-plane"] == null and (.spec.unschedulable != true)) | .metadata.name') [[ ${#workers[@]} == 2 ]] @@ -24,8 +28,8 @@ cleanup() { k taint node "$node" "$key:NoSchedule-" >/dev/null || true k label node "$node" "$key-" >/dev/null || true done - k delete gameserver "$game" -n "$ns" --ignore-not-found --wait=false - k delete gametemplate "$game" --ignore-not-found --wait=false + k delete gameserver "$local_game" "$csi_game" -n "$ns" --ignore-not-found --wait=false + k delete gametemplate "$local_game" --ignore-not-found --wait=false k patch "$op" -n "$panel" --type=json --patch-file "$work/restore.json" >/dev/null k rollout status "$op" -n "$panel" --timeout=3m } @@ -52,6 +56,15 @@ docker build --quiet -t "$image" "$work" kind load docker-image --name "$cluster" "$image" k label node "$games" "$key=games" k label node "$aux" "$key=management" +# Install the upstream Kubernetes CSI distributed reference driver only in +# the disposable kind cluster. The external provisioner and driver manifests +# come from immutable revisions. Game Pods still mount ordinary PVCs. +csi_base=https://raw.githubusercontent.com/kubernetes-csi/csi-driver-host-path/fd04e6e0ba610170ec488864ded0df919986ebdf/deploy/kubernetes-distributed/hostpath +k apply -f https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/12a344a40072d655cb9374a93483305f1a1be557/deploy/kubernetes/rbac.yaml +for component in driverinfo plugin storageclass-fast; do + k apply -f "$csi_base/csi-hostpath-$component.yaml" +done +k -n default rollout status daemonset/csi-hostpathplugin --timeout=4m # Provision the node-bound game claim before adding any taint to the worker. # kind's local-path storage helper does not tolerate arbitrary custom taints. k apply -f - </dev/null)" ]]; } wait_for has_world -marker=$(world) -claim=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}') -pv=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.spec.volumeName}') -k get pv "$pv" -o json | jq -e '.spec.nodeAffinity.required.nodeSelectorTerms | length > 0' >/dev/null +game=$csi_game +wait_for has_world +game=$local_game policy=$(jq -nc --arg key "$key" '{poolLabel:$key,poolValue:"games",minCPURequest:"10m",maxCPURequest:"1",minMemoryRequest:"8Mi",maxMemoryLimit:"32Mi"}') jq -nc --arg key "$key" '{spec:{template:{spec:{nodeSelector:{($key):"management"},tolerations:[{key:$key,operator:"Equal",value:"management",effect:"NoSchedule"}]}}}}' > "$work/pool.json" k -n "$panel" patch "$op" --type=merge --patch-file "$work/pool.json" @@ -115,9 +136,8 @@ for provider in frp tailscale playit; do wait_for provider_ready echo "$provider relay ready on $aux" done -relay=$(pod_on tunnel "$aux") -k cordon "$aux" -k -n "$ns" delete pod "$relay" --wait=false +k drain "$aux" --pod-selector="app.kubernetes.io/name=gameplane-tunnel,app.kubernetes.io/instance=$game" \ + --ignore-daemonsets --delete-emptydir-data --timeout=3m unschedulable() { k -n "$ns" get pods -l "app.kubernetes.io/name=gameplane-tunnel,app.kubernetes.io/instance=$game" -o json | jq -e 'any(.items[]; .metadata.deletionTimestamp==null and any(.status.conditions[]?; .type=="PodScheduled" and .reason=="Unschedulable"))' >/dev/null @@ -127,33 +147,57 @@ k uncordon "$aux" wait_for pod_on tunnel "$aux" relay=$(pod_on tunnel "$aux") nodeIP=$(k get node "$games" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') -for mode in ClusterIP NodePort LoadBalancer Hostport; do - patch_game "$(jq -nc --arg mode "$mode" '{spec:{networking:{expose:$mode}}}')" - # Wait for this exposure's desired game before parking it. - mode_ready() { - service_type=$mode; [[ "$mode" != Hostport ]] || service_type=ClusterIP - [[ "$(k -n "$ns" get service "$game" -o jsonpath='{.spec.type}')" == "$service_type" ]] && - k -n "$ns" get sts "$game" -o json | jq -e --arg mode "$mode" '(.status.readyReplicas==1) and - (([.spec.template.spec.containers[]|select(.name=="game")|.ports[]|.hostPort // 0]|max > 0)==($mode=="Hostport"))' >/dev/null - } - wait_for mode_ready - k -n "$ns" annotate gameserver "$game" "gameplane.local/idle-asleep-since=$(date -u +%FT%TZ)" --overwrite - endpoint=$game.$ns.svc:12345; worker=$aux - if [[ "$mode" == Hostport ]]; then - worker=$games - endpoint=$nodeIP:12345 - elif [[ "$mode" != ClusterIP ]]; then - endpoint=$nodeIP:$(k -n "$ns" get service "$game" -o jsonpath='{.spec.ports[0].nodePort}') +exercise_modes() { + local marker claim pv mode worker endpoint service_type + marker=$(world) + claim=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}') + pv=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.spec.volumeName}') + k get pv "$pv" -o json | jq -e '.spec.nodeAffinity.required.nodeSelectorTerms | length > 0' >/dev/null + if [[ "$game" == "$csi_game" ]]; then + k get pv "$pv" -o json | jq -e '.spec.csi.driver=="hostpath.csi.k8s.io" and (.spec.csi.volumeHandle|length>0)' >/dev/null fi - wait_for pod_on waker "$worker" - parked() { - [[ "$(k -n "$ns" get sts "$game" -o jsonpath='{.spec.replicas}')" == 0 && - -z "$(k -n "$ns" get pod "$game-0" --ignore-not-found -o name)" ]] - } - wait_for parked - join_world() { [[ "$(k -n "$ns" exec "$relay" -- wget -T 6 -qO- "http://$endpoint/world" 2>/dev/null)" == "$marker" ]]; } - wait_for join_world - [[ "$(k -n "$ns" get pod "$game-0" -o jsonpath='{.spec.nodeName}')" == "$games" ]] - [[ "$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}')" == "$claim" ]] - echo "$mode waker=$worker relay=$aux game=$games preserved PVC=$claim world=$marker" -done + for mode in ClusterIP NodePort LoadBalancer Hostport; do + patch_game "$(jq -nc --arg mode "$mode" '{spec:{networking:{expose:$mode}}}')" + # Wait for this exposure's desired game before parking it. + mode_ready() { + service_type=$mode; [[ "$mode" != Hostport ]] || service_type=ClusterIP + [[ "$(k -n "$ns" get service "$game" -o jsonpath='{.spec.type}')" == "$service_type" ]] && + k -n "$ns" get sts "$game" -o json | jq -e --arg mode "$mode" '(.status.readyReplicas==1) and + (([.spec.template.spec.containers[]|select(.name=="game")|.ports[]|.hostPort // 0]|max > 0)==($mode=="Hostport"))' >/dev/null + } + wait_for mode_ready + k -n "$ns" annotate gameserver "$game" "gameplane.local/idle-asleep-since=$(date -u +%FT%TZ)" --overwrite + endpoint=$game.$ns.svc:12345; worker=$aux + if [[ "$mode" == Hostport ]]; then + worker=$games + endpoint=$nodeIP:12345 + elif [[ "$mode" == NodePort ]]; then + endpoint=$nodeIP:$(k -n "$ns" get service "$game" -o jsonpath='{.spec.ports[0].nodePort}') + elif [[ "$mode" == LoadBalancer ]]; then + lb_assigned() { [[ -n "$(k -n "$ns" get service "$game" -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]]; } + wait_for lb_assigned + endpoint=$(k -n "$ns" get service "$game" -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):12345 + fi + wait_for pod_on waker "$worker" + parked() { + [[ "$(k -n "$ns" get sts "$game" -o jsonpath='{.spec.replicas}')" == 0 && + -z "$(k -n "$ns" get pod "$game-0" --ignore-not-found -o name)" ]] + } + wait_for parked + join_world() { [[ "$(k -n "$ns" exec "$relay" -- wget -T 6 -qO- "http://$endpoint/world" 2>/dev/null)" == "$marker" ]]; } + wait_for join_world + [[ "$(k -n "$ns" get pod "$game-0" -o jsonpath='{.spec.nodeName}')" == "$games" ]] + [[ "$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}')" == "$claim" ]] + echo "$mode waker=$worker relay=$aux game=$games preserved PVC=$claim world=$marker" + done +} +exercise_modes +# Release the physical host port on this worker before the CSI-backed game +# exercises the same public endpoint. +patch_game '{"spec":{"networking":{"expose":"ClusterIP"}}}' +hostport_released() { + k -n "$ns" get pod "$game-0" -o json | jq -e '[.spec.containers[]|.ports[]?|.hostPort // 0] | all(. == 0)' >/dev/null +} +wait_for hostport_released +game=$csi_game +exercise_modes From 3e0d28115ccd310657692a47e06abe76af368cfc Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 03:20:48 -0500 Subject: [PATCH 4/4] test(operator): qualify node-bound worlds through game worker drain Signed-off-by: Baylem --- docs/production-placement.md | 16 +++++++------ test/e2e/auxiliary-placement.sh | 41 ++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/docs/production-placement.md b/docs/production-placement.md index 670f6e33e..5f8d54a3e 100644 --- a/docs/production-placement.md +++ b/docs/production-placement.md @@ -161,13 +161,15 @@ selectors, taints, affinity, or exhausted resources. Restore an eligible worker for the same pool (and the original endpoint for Hostport). The disposable kind management drill runs the operator on a tainted management -worker, binds a populated local-path PVC on a separate game worker, parks -the game and wakes it over all four exposure modes. It reads the original -world marker after each handoff. It also verifies the scheduler's diagnostic -after evicting a relay from its only eligible worker. Each provider uses a -local health responder so this placement drill needs no external accounts. -The LoadBalancer case exercises its allocated NodePort. Test a real external -address and the site's CSI driver on the deployment topology before rollout. +worker and provisions local-path and topology-bound CSI game volumes on a +separate game worker. It reads each world's saved marker after a wake through +ClusterIP, NodePort, the MetalLB address and Hostport. It drains each pool +in turn. The relay and games must report an unschedulable replacement while +their own worker is cordoned. After the game worker returns, the drill records +recovery time and verifies both PVC and PV identities and saved markers. +Each tunnel provider uses a local health responder. The CSI lane uses the +upstream distributed host-path reference driver. Qualify the site's external +network path and production CSI driver on its deployment topology. A priorityClassName must name an existing PriorityClass. Unsatisfied hard selectors, taints or topology diff --git a/test/e2e/auxiliary-placement.sh b/test/e2e/auxiliary-placement.sh index 4a93ec1d3..bbe453f0a 100644 --- a/test/e2e/auxiliary-placement.sh +++ b/test/e2e/auxiliary-placement.sh @@ -8,7 +8,7 @@ context="kind-${cluster}" k() { kubectl --context="$context" "$@"; } ns=gameplane-games panel=gameplane-system -game=aux-worker-drill +game="aux-worker-drill" local_game=$game csi_game=${game}-csi key=gameplane.local/aux-drill @@ -24,6 +24,7 @@ k get nodes -l node-role.kubernetes.io/control-plane -o json | jq -e 'any(.items k get "$op" -n "$panel" -o json | jq '[{op:"replace",path:"/spec/template",value:.spec.template}]' > "$work/restore.json" cleanup() { k uncordon "$aux" >/dev/null || true + k uncordon "$games" >/dev/null || true for node in "$games" "$aux"; do k taint node "$node" "$key:NoSchedule-" >/dev/null || true k label node "$node" "$key-" >/dev/null || true @@ -201,3 +202,41 @@ hostport_released() { wait_for hostport_released game=$csi_game exercise_modes + +# Drain only the two populated games in this shared test cluster. Both PVs +# are node-bound. Keep the separate relay pool available during the drain. +declare -A before_pod before_world before_volume +volume_identity() { + local binding pv uid + binding=$(k -n "$ns" get pvc "$game-data" -o jsonpath='{.metadata.uid}/{.spec.volumeName}') || return 1 + pv=${binding#*/} + [[ -n "$pv" ]] || return 1 + uid=$(k get pv "$pv" -o jsonpath='{.metadata.uid}') || return 1 + [[ -n "$uid" ]] || return 1 + printf '%s/%s' "$binding" "$uid" +} +for game in "$local_game" "$csi_game"; do + before_pod[$game]=$(k -n "$ns" get pod "$game-0" -o jsonpath='{.metadata.uid}') + before_world[$game]=$(world) + before_volume[$game]=$(volume_identity) +done +drain_started=$SECONDS +k drain "$games" --pod-selector="app.kubernetes.io/name=gameplane-game,app.kubernetes.io/instance in ($local_game,$csi_game)" \ + --ignore-daemonsets --delete-emptydir-data --timeout=3m +game_pending() { + k -n "$ns" get pod "$game-0" -o json | jq -er --arg old "${before_pod[$game]}" ' + select(.metadata.uid!=$old and .metadata.deletionTimestamp==null and (.spec.nodeName // "")=="") | + .status.conditions[]? | select(.type=="PodScheduled" and .status=="False" and .reason=="Unschedulable") | .message' +} +for game in "$local_game" "$csi_game"; do + wait_for game_pending + [[ "$(volume_identity)" == "${before_volume[$game]}" ]] +done +game=$local_game +pod_on tunnel "$aux" >/dev/null +k uncordon "$games" +for game in "$local_game" "$csi_game"; do + wait_for pod_on game "$games" + [[ "$(volume_identity)" == "${before_volume[$game]}" && "$(world)" == "${before_world[$game]}" ]] + echo "$game recovered on $games $((SECONDS-drain_started))s after drain began, PVC/PV=${before_volume[$game]} world=${before_world[$game]}" +done