feat(operator): enforce tenant resource bounds and game-worker placement

.github/workflows/ci.yaml

@@ -1869,7 +1869,7 @@ jobs:
             job_timeout: 60 # Boots two-node kind, installs the chart and exercises repeated Hostport sleep/wake handoffs.
           - bucket: management-placement
             parallel: 1
-            test_timeout: 15m
+            test_timeout: 30m
             job_timeout: 60 # Boots management kind, installs the chart and checks placement/disruption behavior.
           - bucket: backup-preparation
             parallel: 1

changelog/unreleased/903-game-worker-policy.md

@@ -0,0 +1,8 @@
+### Game worker placement and resource limits
+
+Administrators can reserve game workers with a node label and taint, bound CPU
+requests and memory reservations and limits, and set per-namespace quotas.
+GameServer overrides retain the template quantities for resources they omit.
+When the policy takes over a legacy game, the operator drains the existing Pod
+before applying placement and resource changes. A verified running revision
+uses its saved stop sequence. An unversioned game uses its Pod termination grace.

charts/gameplane/templates/game-worker-admission.yaml

@@ -0,0 +1,59 @@
+{{- if and (eq (include "gameplane.operatorEnabled" .) "true") (ne .Values.operator.gameWorkloadPolicy nil) }}
+{{- $p := .Values.operator.gameWorkloadPolicy }}
+{{- $key := required "gameWorkloadPolicy.poolLabel is required" $p.poolLabel | toJson }}
+{{- $value := required "gameWorkloadPolicy.poolValue is required" $p.poolValue | toJson }}
+{{- $name := printf "%s.%s.game-workers" .Release.Name .Release.Namespace }}
+# An existing StatefulSet can recreate its Pod before the operator sees a
+# changed policy. Enforce the scheduler constraint and bounds on that path too.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: {{ $name }}
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: [v1]
+        operations: [CREATE]
+        resources: [pods]
+      - apiGroups: [""]
+        apiVersions: [v1]
+        operations: [UPDATE]
+        resources: [pods/resize]
+  validations:
+    - expression: >-
+        has(object.spec.nodeSelector) && {{ $key }} in object.spec.nodeSelector &&
+        object.spec.nodeSelector[{{ $key }}] == {{ $value }}
+      message: "game pods must select the configured game worker pool"
+    - expression: >-
+        (request.operation != "CREATE" || !has(object.spec.nodeName) || object.spec.nodeName == "") &&
+        (!has(object.spec.tolerations) || object.spec.tolerations.all(t, has(t.key) && t.key != ""))
+      message: "game pods must use the scheduler and may not tolerate every taint"
+    - expression: >-
+        object.spec.containers.exists(c, c.name == "game" &&
+        has(c.resources) && has(c.resources.requests) && has(c.resources.limits) &&
+        "cpu" in c.resources.requests && "memory" in c.resources.requests && "memory" in c.resources.limits &&
+        quantity(string(c.resources.requests["cpu"])).compareTo(quantity({{ required "minCPURequest is required" $p.minCPURequest | toString | toJson }})) >= 0 &&
+        quantity(string(c.resources.requests["cpu"])).compareTo(quantity({{ required "maxCPURequest is required" $p.maxCPURequest | toString | toJson }})) <= 0 &&
+        quantity(string(c.resources.requests["memory"])).compareTo(quantity({{ required "minMemoryRequest is required" $p.minMemoryRequest | toString | toJson }})) >= 0 &&
+        quantity(string(c.resources.limits["memory"])).compareTo(quantity({{ required "maxMemoryLimit is required" $p.maxMemoryLimit | toString | toJson }})) <= 0)
+      message: "game containers must meet the configured CPU request and memory request/limit bounds"
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: {{ $name }}
+spec:
+  policyName: {{ $name }}
+  validationActions: [Deny]
+  matchResources:
+    namespaceSelector:
+      matchExpressions:
+        - key: kubernetes.io/metadata.name
+          operator: In
+          values: {{ include "gameplane.operatorGameNamespaces" . }}
+    objectSelector:
+      matchLabels:
+        app.kubernetes.io/name: gameplane-game
+{{- end }}

charts/gameplane/templates/operator.yaml

@@ -409,6 +409,9 @@ spec:
             {{- if .Values.operator.agentLogLevel }}
             - --agent-log-level={{ .Values.operator.agentLogLevel }}
             {{- end }}
+            {{- if ne .Values.operator.gameWorkloadPolicy nil }}
+            - {{ printf "--game-workload-policy=%s" (mustToJson .Values.operator.gameWorkloadPolicy) | quote }}
+            {{- end }}
             {{- if and .Values.operator.gameDataStorage .Values.operator.gameDataStorage.storageClassName }}
             - --game-data-storage-class={{ .Values.operator.gameDataStorage.storageClassName }}
             {{- end }}

charts/gameplane/templates/tenant-quotas.yaml

@@ -0,0 +1,29 @@
+{{- if eq (include "gameplane.operatorEnabled" .) "true" }}
+{{- $managed := include "gameplane.operatorGameNamespaces" . | fromJsonArray }}
+{{- range $ns, $config := .Values.operator.tenantQuotas }}
+{{- if not (has $ns $managed) }}{{ fail (printf "operator.tenantQuotas namespace %s must be a managed game namespace" $ns) }}{{ end }}
+---
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: {{ $.Release.Name }}-tenant-budget
+  namespace: {{ $ns }}
+  labels: {{- include "gameplane.labels" $ | nindent 4 }}
+spec:
+  hard: {{- required "each operator.tenantQuotas entry requires hard resources" $config.hard | toYaml | nindent 4 }}
+{{- with $config.defaultContainerMemoryLimit }}
+---
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: {{ $.Release.Name }}-container-defaults
+  namespace: {{ $ns }}
+  labels: {{- include "gameplane.labels" $ | nindent 4 }}
+spec:
+  limits:
+    - type: Container
+      default:
+        memory: {{ . | quote }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/gameplane/values-production.example.yaml

@@ -2,6 +2,25 @@
 # applying this profile; see docs/production-placement.md. Never remove or
 # tolerate control-plane taints just to satisfy these selectors.
 operator:
+  gameWorkloadPolicy:
+    poolLabel: gameplane.local/pool
+    poolValue: games
+    minCPURequest: 100m
+    maxCPURequest: "4"
+    minMemoryRequest: 128Mi
+    maxMemoryLimit: 8Gi
+  # Budget for the default games namespace. Use a separate managed namespace
+  # and independently sized quota for each tenant in a shared installation.
+  tenantQuotas:
+    gameplane-games:
+      defaultContainerMemoryLimit: 512Mi
+      hard:
+        requests.cpu: "12"
+        requests.memory: 24Gi
+        limits.memory: 32Gi
+        pods: "24"
+        persistentvolumeclaims: "12"
+        requests.storage: 200Gi
   replicas: 2
   leaderElect: true
   nodeSelector: &managementNodes { gameplane.local/pool: management }

charts/gameplane/values.yaml

@@ -85,6 +85,12 @@ operator:
   resources:
     requests: { cpu: 100m, memory: 64Mi }
     limits:   { cpu: 500m, memory: 256Mi }
+  # Optional game worker label/NoSchedule taint and per-game tenant bounds.
+  # See values-production.example.yaml and docs/production-placement.md.
+  gameWorkloadPolicy: null
+  # ResourceQuota by managed game namespace. Each entry accepts hard and
+  # an optional defaultContainerMemoryLimit for containers without a limit.
+  tenantQuotas: {}
   # Complete auxiliary ResourceRequirements overrides. Null/missing uses binary
   # defaults, including on --reuse-values upgrades. {} (or requests: {}, limits:
   # {}) clears defaults. No new CPU/memory limits are set by default.

docs/production-placement.md

@@ -53,8 +53,9 @@ requests or limits, replacing the install agent object wholesale. An image-only
 agent override keeps install defaults. Kubernetes may omit empty maps in a
 template, so use explicit zero CPU/memory requests for a per-template zero
 reservation, or the install-level empty object to clear all defaults. Game
-container resources continue to use the existing template defaults and complete
-`GameServer.spec.resources` override. Game placement fields are unchanged.
+container requests and limits merge per resource. For example, a server
+overriding only `requests.cpu` inherits both memory values and any template
+CPU limit. A tier with no CPU limit in either object keeps CPU bursting.
 
 Changing resources changes generated Pod templates. Existing game StatefulSets,
 sentinels and tunnels roll according to their normal reconciliation policy;
@@ -106,7 +107,36 @@ kubectl label node game-a game-b gameplane.local/pool=games
 kubectl taint node game-a game-b gameplane.local/pool=games:NoSchedule
 ```
 
-Use existing GameServer placement fields for game workers:
+The production example enables `operator.gameWorkloadPolicy`. The operator
+adds the chosen pool selector and its exact `Equal` / `NoSchedule` toleration
+to each game Pod. A server can narrow placement with additional node labels
+or affinity. A conflicting pool selector or an extra toleration produces a
+failed Ready condition. A pool with no schedulable workers leaves games Pending.
+Keep permission to change pool labels and taints with cluster administrators.
+Enabling the policy for a legacy game can briefly interrupt play. The operator
+drains its existing Pod before moving it to the configured pool. Games with
+a verified running template use its stop sequence. Unversioned games use the
+termination grace already configured on their Pods.
+With the NodeRestriction admission plugin, a label key such as
+`gameplane.node-restriction.kubernetes.io/pool` also prevents a kubelet from
+labeling itself as a game worker. Set the configured key and worker taint to match.
+
+The policy validates effective game resources before workload creation and
+template adoption. The CPU request must fall between the configured bounds.
+The memory request must meet its floor, and the memory limit must cover the
+request and fit the configured ceiling. Set a CPU limit in the template or
+server only for tiers that require one. Identify tiers by their measured
+reservations and ceilings, since sidecar requests also influence Pod QoS.
+
+The chart installs a Kubernetes v1 ValidatingAdmissionPolicy with the profile.
+It checks newly created game Pods and in-place resizes in the managed game
+namespaces, including replacements from pre-upgrade StatefulSets. Existing
+running Pods keep their allocation until replaced. Before enabling the profile,
+inventory live requests, limits, selectors, tolerations and PVC topology.
+Correct incompatible templates or server overrides and roll games during a
+maintenance window. A `null` policy keeps administrator enforcement disabled.
+
+Without a policy, a server can request its own placement:
 
 ```yaml
 spec:
@@ -127,6 +157,23 @@ Check API/receiver PVC node affinity before selecting management workers, and
 game PVC topology before backups or moving game pods. Restic scheduling and
 source preparation are separate from management placement.
 
+## Tenant namespace quotas
+
+Provision a namespace for each tenant and add it to the operator's managed
+game namespaces. Configure API/RBAC grants for the same tenant boundary.
+Set `operator.tenantQuotas.<namespace>.hard` for `requests.cpu`,
+`requests.memory`, `limits.memory`, `pods`, `persistentvolumeclaims` and
+`requests.storage`. The production values illustrate a budget for the default
+games namespace. Replace that key if you change `gamesNamespace`. Tenants
+sharing a namespace share its quota.
+
+When using `limits.memory`, give supporting containers a memory limit.
+`defaultContainerMemoryLimit` installs a LimitRange that supplies a default
+for containers without an explicit memory limit. Size it for backup workers,
+sidecars and initializers as well as games. A quota denial appears in
+StatefulSet or Job events. Reserve quota for active backups, restore PVCs,
+surge Pods and starts. Leave `limits.cpu` out of quotas for uncapped CPU tiers.
+
 ## Sizing and controlled-drain checks
 
 Collect per-container CPU/memory, pod requests, Pending events, game tick/query
@@ -136,6 +183,28 @@ the available Prometheus metrics. Include headroom for one failed management
 node, game startup, repository maintenance and replacement pods. No fixed server
 count follows from these request values.
 
+Qualify each production tier on the intended node and storage classes. Record
+the concurrent game count, tick/query latency p95 and p99, CPU throttling,
+memory working sets and completed backup duration at steady state. Repeat
+while a neighboring game allocates to its memory limit, a cold game starts and
+the expected peak backup concurrency runs. Record the OOM termination and
+verify the neighboring game stays within its latency target. Cordon and drain
+one game worker and record replacement placement, PVC attachment, Pending
+events and recovery time. For each eligible worker, compare remaining
+allocatable CPU and memory after requests to the largest game start plus its
+sidecars, and reserve capacity for the backup burst after one worker is lost.
+Record each tier's accepted tenant quotas, node count and latency target from
+that run before offering it to tenants.
+
+The disposable `management-placement` CI bucket runs a Minecraft Java
+1.21.4 flat world alongside the replacement tenant Pod and a Restic worker
+on the sole remaining eligible node. It records Minecraft's own `tick query`
+p95/p99, the worker's allocatable and requested resources, the tenant OOM
+exit and the time to complete and verify the backup. It checks the neighbor's
+Pod UID and container restart counts across the event. The small fixture's
+100 ms p95 and 250 ms p99 budgets apply to this CI run. Qualify production
+tiers with representative player counts, worlds and storage.
+
 In a disposable multi-worker environment, apply the example and verify every
 management pod's node labels. Temporarily set a nonexistent selector and verify
 Pending/FailedScheduling, then restore it. Cordon one management worker, evict

operator/cmd/main.go

@@ -163,6 +163,7 @@ func main() {
 		backupCSICaptureTimeout          time.Duration
 		backupHardSameNode               bool
 		agentResourcesJSON               string
+		gameWorkloadPolicyJSON           string
 		sentinelResourcesJSON            string
 		tunnelResourcesJSON              string
 		backupResourcesJSON              string
@@ -171,6 +172,7 @@ func main() {
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election.")
 	flag.StringVar(&agentResourcesJSON, "agent-resources", "", "JSON ResourceRequirements for agents and config init containers; default requests 25m/64Mi, {} clears defaults.")
+	flag.StringVar(&gameWorkloadPolicyJSON, "game-workload-policy", "", "JSON game worker pool and per-game resource bounds. Unset disables policy enforcement.")
 	flag.StringVar(&sentinelResourcesJSON, "sentinel-resources", "", "JSON ResourceRequirements for sentinels; default requests 25m/32Mi, {} clears defaults.")
 	flag.StringVar(&tunnelResourcesJSON, "tunnel-resources", "", "JSON ResourceRequirements for tunnel relays; default requests 25m/64Mi, {} clears defaults.")
 	flag.StringVar(&backupResourcesJSON, "backup-resources", "", "JSON ResourceRequirements for restic backup/init, restore and cleanup workers; default backup/restore requests 100m/128Mi, {} clears defaults.")
@@ -313,6 +315,11 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 	setupLog := ctrl.Log.WithName("setup")
 
+	gameWorkloadPolicy, err := controller.ParseGameWorkloadPolicy(gameWorkloadPolicyJSON)
+	if err != nil {
+		setupLog.Error(err, "invalid --game-workload-policy")
+		os.Exit(1)
+	}
 	agentResources, err := controller.ParseAuxiliaryResources(agentResourcesJSON)
 	if err != nil {
 		setupLog.Error(err, "invalid --agent-resources")
@@ -498,6 +505,7 @@ func main() {
 		ProbeIngressFromCIDRs:       probeIngressFromCIDR,
 		CaptureEnabled:              captureEnabled,
 		RestrictedWorkloads:         restrictedWorkloads,
+		GameWorkloadPolicy:          gameWorkloadPolicy,
 		AgentTokenExpirationSeconds: agentTokenExpirationSeconds,
 		DiagnosticsNamespace:        diagnosticsNamespace,
 		CaptureSidecarImage:         captureSidecarImage,