From 8be751286210623685edc23c0287e1e79d5b9460 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 23:37:47 -0500 Subject: [PATCH 1/5] fix(rbac): scope native administration and snapshot permissions by mode Signed-off-by: Baylem --- .github/workflows/ci.yaml | 2 +- api/cmd/main.go | 12 +- api/internal/handlers/clusters.go | 9 +- api/specs.md | 2 + charts/gameplane/templates/_helpers.tpl | 26 +- .../gameplane/templates/api-credentials.yaml | 12 + charts/gameplane/templates/api.yaml | 49 ++-- .../templates/native-admin-rbac.yaml | 49 ++++ charts/gameplane/templates/operator.yaml | 13 +- .../templates/validate-native-rbac.yaml | 15 ++ charts/gameplane/values.yaml | 17 ++ charts/tests/check_standalone.py | 36 ++- docs/native-rbac.md | 44 ++++ docs/security.md | 2 + operator/cmd/main.go | 24 +- operator/config/rbac/role.yaml | 8 - .../internal/controller/backup_controller.go | 3 +- operator/internal/secretscope/cache.go | 4 +- operator/specs.md | 2 +- test/e2e/game_admission_e2e_test.go | 11 +- test/e2e/native_rbac_e2e_test.go | 235 ++++++++++++++++++ test/e2e/upgrade_e2e_test.go | 8 + 22 files changed, 523 insertions(+), 60 deletions(-) create mode 100644 charts/gameplane/templates/native-admin-rbac.yaml create mode 100644 charts/gameplane/templates/validate-native-rbac.yaml create mode 100644 docs/native-rbac.md create mode 100644 test/e2e/native_rbac_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ce0fa057..9fb202e12 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2179,7 +2179,7 @@ jobs: cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ - go test -tags=e2e -timeout 25m -v -run "$RUN_REGEX" ./... + go test -tags=e2e -timeout 35m -v -run "$RUN_REGEX" ./... - name: dump cluster state on failure (e2e-upgrade) if: failure() uses: ./.github/actions/dump-cluster-state diff --git a/api/cmd/main.go b/api/cmd/main.go index a4a08e944..532b4b876 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -285,7 +285,9 @@ func main() { // with remote cluster connections. Single-cluster installs only have the // "local" cluster (the control plane); this watcher loads remote clusters // from kubeconfigs stored as Secrets. - go kube.WatchClusters(ctx, k8s, reg, credentialsNS) + if cfg.remoteClusters { + go kube.WatchClusters(ctx, k8s, reg, credentialsNS) + } // Optional Steam display-name lookup for player lists. Off unless the // key is set; the resolver is then the only holder of the key and of @@ -394,7 +396,11 @@ func main() { handlers.MountAuthProviderSecrets(p, k8s, credentialsNS) handlers.MountCluster(p, reg, store, Version, cfg.clusterOps, cfg.updateChannel) handlers.MountClusterActions(p, k8s, cfg.clusterOps, cfg.clusterExternalAddress) - handlers.MountClusters(p, reg, k8s, credentialsNS) + clusterManagement := k8s + if !cfg.remoteClusters { + clusterManagement = nil + } + handlers.MountClusters(p, reg, clusterManagement, credentialsNS) handlers.MountInstallation(p, cfg.standalone, Version) handlers.MountEvents(p, reg) handlers.MountDestinations(p, reg) @@ -516,6 +522,7 @@ func main() { } type config struct { + remoteClusters bool standalone bool panelKeyFile string panelKeyProvisioned bool @@ -673,6 +680,7 @@ func (c *config) bindFlags(fs *flag.FlagSet) { fs.IntVar(&c.captureDefaultMaxDurationS, "capture-default-max-duration", envOrInt("GAMEPLANE_CAPTURE_DEFAULT_MAX_DURATION", 300), "default max capture duration in seconds") c.captureDefaultMaxSizeBytes = envOrInt64("GAMEPLANE_CAPTURE_DEFAULT_MAX_SIZE", 5368709120) fs.BoolVar(&c.standalone, "standalone", envOr("GAMEPLANE_STANDALONE", "") == "true", "run without a local Kubernetes cluster") + fs.BoolVar(&c.remoteClusters, "remote-clusters", true, "load remote cluster credentials and connect to registered clusters") fs.StringVar(&c.panelKeyFile, "panel-key-file", envOr("GAMEPLANE_PANEL_KEY_FILE", "/data/panel.key"), "persistent standalone credential encryption key file") fs.BoolVar(&c.panelKeyProvisioned, "panel-key-provisioned", envOr("GAMEPLANE_PANEL_KEY_PROVISIONED", "") == "true", "require an existing read-only standalone credential key; never generate it") fs.StringVar(&c.remoteAllowedCIDRs, "remote-allowed-cidrs", envOr("GAMEPLANE_REMOTE_ALLOWED_CIDRS", ""), "optional comma-separated CIDRs allowed for standalone Kubernetes and gateway connections") diff --git a/api/internal/handlers/clusters.go b/api/internal/handlers/clusters.go index 110b8cea9..7b7cc6369 100644 --- a/api/internal/handlers/clusters.go +++ b/api/internal/handlers/clusters.go @@ -28,11 +28,14 @@ import ( "github.com/GameplanePanel/gameplane/api/internal/messages" ) -// MountClusters wires /clusters onto the supplied router. +// MountClusters registers only local discovery when no remote client is supplied. func MountClusters(r chi.Router, reg *kube.Registry, k *kube.Client, ns string) { h := clustersHandler{reg: reg, k: k, namespace: ns} r.Route("/clusters", func(r chi.Router) { r.Get("/", h.list) + if k == nil { + return + } r.Post("/", h.create) r.Delete("/{name}", h.delete) r.Put("/{name}/kubeconfig", h.replaceKubeconfig) @@ -122,6 +125,10 @@ func (h clustersHandler) list(w http.ResponseWriter, req *http.Request) { out.Items = append(out.Items, clusterRegistryView{Name: local, Phase: "Healthy", CanViewInventory: u.Can("cluster:read", true, local, "")}) } + if h.k == nil { + writeJSON(w, out) + return + } // Persisted registrations remain discoverable when a kubeconfig cannot // load; the client registry alone would silently drop those clusters. registrations, err := h.k.Clusters().List(req.Context(), metav1.ListOptions{}) diff --git a/api/specs.md b/api/specs.md index 446c71ce2..cfda2cd2f 100644 --- a/api/specs.md +++ b/api/specs.md @@ -475,6 +475,8 @@ A submitted stylesheet is rejected with 400 and a message naming the offending r ### RBAC roles (three built-in + custom) +Kubernetes authorizes the native panel's ServiceAccount separately from the user's Gameplane role. In local-only mode, `--remote-clusters=false` skips remote credential discovery, exposes only authorized local cluster discovery and omits remote registration and gateway-management routes. Native Helm grants and the upgrade contract are specified in [the Kubernetes permission matrix](../docs/native-rbac.md). + **Built-in roles:** - **admin:** wildcard permission `*`; full access to all resources and config - **operator:** read/write servers, backups, schedules, templates; read modules, destinations, cluster, roles (modules install/upgrade/uninstall requires `modules:manage`, seeded only to admin — see `api/internal/db/migrations/sqlite/003_roles.sql`) diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index ad18319b0..483c16ec0 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -182,6 +182,26 @@ mergeOverwrite preserves explicit zero/empty values so validation can reject the {{- uniq $namespaces | toJson -}} {{- end -}} +{{- /* helm upgrade --reuse-values can omit a values map introduced by a +new chart. Preserve the privileges and watch behavior of that installation. */ -}} +{{- define "gameplane.nativeRBAC" -}} +{{- $defaults := dict "catalogAdmin" true "remoteClusters" true "gameSecretManagement" true "podAttach" true "operatorStopAttach" true "snapshotContentReclamation" true "runtimeSecretNames" (list) "additionalAPIGameNamespaces" (list) -}} +{{- $overrides := dict -}} +{{- if hasKey .Values "nativeRBAC" }}{{ $overrides = .Values.nativeRBAC }}{{ end -}} +{{- if not (kindIs "map" $overrides) }}{{ fail "nativeRBAC must be a map" }}{{ end -}} +{{- range $key := keys $overrides -}} +{{- if not (hasKey $defaults $key) }}{{ fail (printf "unknown nativeRBAC capability %q" $key) }}{{ end -}} +{{- end -}} +{{- $config := mergeOverwrite $defaults $overrides -}} +{{- range $key := list "catalogAdmin" "remoteClusters" "gameSecretManagement" "podAttach" "operatorStopAttach" "snapshotContentReclamation" -}} +{{- if not (kindIs "bool" (index $config $key)) }}{{ fail (printf "nativeRBAC.%s must be a boolean" $key) }}{{ end -}} +{{- end -}} +{{- range $key := list "additionalAPIGameNamespaces" "runtimeSecretNames" -}} +{{- if not (kindIs "slice" (index $config $key)) }}{{ fail (printf "nativeRBAC.%s must be a list" $key) }}{{ end -}} +{{- end -}} +{{- $config | toJson -}} +{{- end -}} + {{- /* Both native components resolve remote-cluster credentials here. Keep the existing operator override for installations with a dedicated namespace. */ -}} {{- define "gameplane.credentialsNamespace" -}} @@ -201,8 +221,12 @@ the existing operator override for installations with a dedicated namespace. */ {{- end -}} {{- define "gameplane.operatorSecretNamespaces" -}} +{{- $rbac := include "gameplane.nativeRBAC" . | fromJson -}} {{- $namespaces := include "gameplane.operatorGameNamespaces" . | fromJsonArray -}} -{{- range list (include "gameplane.operatorClusterCredentialsNamespace" .) (.Values.operator.agentCASecretNamespace | default .Release.Namespace) (.Values.operator.moduleNamespace | default .Release.Namespace) -}} +{{- if $rbac.remoteClusters -}} +{{- $namespaces = append $namespaces (include "gameplane.operatorClusterCredentialsNamespace" .) -}} +{{- end -}} +{{- range list (.Values.operator.agentCASecretNamespace | default .Release.Namespace) (.Values.operator.moduleNamespace | default .Release.Namespace) -}} {{- $namespaces = append $namespaces (include "gameplane.namespace" .) -}} {{- end -}} {{- uniq $namespaces | toJson -}} diff --git a/charts/gameplane/templates/api-credentials.yaml b/charts/gameplane/templates/api-credentials.yaml index 5173394b3..dba430782 100644 --- a/charts/gameplane/templates/api-credentials.yaml +++ b/charts/gameplane/templates/api-credentials.yaml @@ -1,3 +1,4 @@ +{{- $rbac := include "gameplane.nativeRBAC" . | fromJson -}} {{- if and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone) }} {{- $namespace := include "gameplane.credentialsNamespace" . }} {{- if not (or .Values.api.credentialsNamespace .Values.operator.controlPlaneNamespace) }} @@ -17,12 +18,23 @@ metadata: namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} rules: + {{- if $rbac.remoteClusters }} - apiGroups: [""] resources: [secrets] verbs: [get, create, update, delete] + {{- else if $rbac.runtimeSecretNames }} + - apiGroups: [""] + resources: [secrets] + verbs: [get] + resourceNames: {{ $rbac.runtimeSecretNames | toJson }} + {{- end }} - apiGroups: [""] resources: [configmaps] + {{- if $rbac.catalogAdmin }} verbs: [get, list, create, update, delete] + {{- else }} + verbs: [get, list] + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index 2dc4b937f..31c9ab8fa 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -1,3 +1,4 @@ +{{- $rbac := include "gameplane.nativeRBAC" . | fromJson -}} {{- if not (eq (include "gameplane.apiEnabled" .) "true") }} {{- $livePVC := lookup "v1" "PersistentVolumeClaim" .Release.Namespace "gameplane-api-data" }} {{- if and $livePVC (ne (index ($livePVC.metadata.annotations | default dict) "helm.sh/resource-policy") "keep") }} @@ -13,10 +14,8 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} {{- if not .Values.api.standalone }} --- -# Cluster-wide READ access: templates (cluster-scoped), nodes, storage -# classes, and read-only watches on Gameplane CRDs for the /events stream. -# Note the absence of any write verb here — mutation permissions are -# bound per-namespace via Role + RoleBinding below. +# Cluster-wide discovery and dashboard reads. The optional catalog and remote +# registration writers have separate Roles and bindings. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -26,18 +25,11 @@ rules: - apiGroups: ["gameplane.local"] resources: [gameservers, gametemplates, backups, backupschedules, restores, modules, modulesources, networkcaptures] verbs: [get, list, watch] - - apiGroups: ["gameplane.local"] - resources: [gametemplates] - verbs: [create, update, patch, delete] # templates are cluster-scoped; admin UI edits them - - apiGroups: ["gameplane.local"] - resources: [modules] - verbs: [create, update, patch, delete] # modules are cluster-scoped; install/upgrade/uninstall from the dashboard - - apiGroups: ["gameplane.local"] - resources: [modulesources] - verbs: [create, update, patch, delete] # source management (add/edit/remove sources) from the dashboard + {{- if $rbac.remoteClusters }} - apiGroups: ["gameplane.local"] resources: [clusters] - verbs: [get, list, watch, create, update, delete] # registration and gateway configuration from the dashboard + verbs: [get, list, watch] + {{- end }} - apiGroups: [""] resources: [nodes, events] verbs: [get, list, watch] @@ -76,16 +68,13 @@ subjects: name: gameplane-api namespace: {{ .Release.Namespace }} --- -# Namespace-scoped WRITE access: limited to the games namespace(s) the -# API is allowed to mutate. Expanding this to additional namespaces -# requires a new RoleBinding here AND adding them to -# GAMEPLANE_EXTRA_NAMESPACES on the API Deployment. +{{- range $ns := uniq (concat (list .Values.gamesNamespace) $rbac.additionalAPIGameNamespaces) }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ .Release.Name }}-api-games - namespace: {{ .Values.gamesNamespace }} - labels: {{- include "gameplane.labels" . | nindent 4 }} + name: {{ $.Release.Name }}-api-games + namespace: {{ $ns }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} rules: - apiGroups: ["gameplane.local"] resources: [gameservers, backups, backupschedules, restores, networkcaptures] @@ -112,9 +101,11 @@ rules: - apiGroups: [""] resources: [pods] verbs: [get] + {{- if $rbac.podAttach }} - apiGroups: [""] resources: [pods/attach] verbs: [get, create] + {{- end }} # The startup-log stream (api/internal/ws/podlogs.go) follows the game # container's stdout via the pod-log subresource so download/config # output is visible before the game's own log file exists. @@ -123,22 +114,25 @@ rules: verbs: [get] # Backup destinations are Secrets in the games namespace # (api/internal/handlers/destinations.go). + {{- if $rbac.gameSecretManagement }} - apiGroups: [""] resources: [secrets] verbs: [get, list, create, update, patch, delete] + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ .Release.Name }}-api-games - namespace: {{ .Values.gamesNamespace }} - labels: {{- include "gameplane.labels" . | nindent 4 }} -roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ .Release.Name }}-api-games } + name: {{ $.Release.Name }}-api-games + namespace: {{ $ns }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $.Release.Name }}-api-games } subjects: - kind: ServiceAccount name: gameplane-api - namespace: {{ .Release.Namespace }} + namespace: {{ $.Release.Namespace }} --- +{{- end }} # System logs are the only namespaced release resources the native API needs. apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -259,6 +253,9 @@ spec: image: {{ include "gameplane.apiImage" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: + {{- if and (not .Values.api.standalone) (not $rbac.remoteClusters) }} + - --remote-clusters=false + {{- end }} {{- if .Values.api.standalone }} - --standalone - {{ printf "--panel-key-file=/keys/%s" ((.Values.api.panelKey | default dict).fileName | default "panel.key") | quote }} diff --git a/charts/gameplane/templates/native-admin-rbac.yaml b/charts/gameplane/templates/native-admin-rbac.yaml new file mode 100644 index 000000000..209da9878 --- /dev/null +++ b/charts/gameplane/templates/native-admin-rbac.yaml @@ -0,0 +1,49 @@ +{{- $rbac := include "gameplane.nativeRBAC" . | fromJson -}} +{{- if and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone) }} +{{- range $role := list + (dict "name" "api-catalog-admin" "enabled" $rbac.catalogAdmin "group" "gameplane.local" "resources" (list "gametemplates" "modules" "modulesources") "verbs" (list "create" "update" "patch" "delete")) + (dict "name" "api-cluster-registration" "enabled" $rbac.remoteClusters "group" "gameplane.local" "resources" (list "clusters") "verbs" (list "create" "update" "delete")) }} +{{- if $role.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $.Release.Name }}-{{ $role.name }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +rules: + - apiGroups: [{{ $role.group | quote }}] + resources: {{ $role.resources | toJson }} + verbs: {{ $role.verbs | toJson }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $.Release.Name }}-{{ $role.name }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ $.Release.Name }}-{{ $role.name }} } +subjects: + - { kind: ServiceAccount, name: gameplane-api, namespace: {{ $.Release.Namespace }} } +{{- end }} +{{- end }} +{{- end }} +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") $rbac.snapshotContentReclamation }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-operator-snapshot-reclaim + labels: {{- include "gameplane.labels" . | nindent 4 }} +rules: + - apiGroups: [snapshot.storage.k8s.io] + resources: [volumesnapshotcontents] + verbs: [delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-operator-snapshot-reclaim + labels: {{- include "gameplane.labels" . | nindent 4 }} +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ .Release.Name }}-operator-snapshot-reclaim } +subjects: + - { kind: ServiceAccount, name: gameplane-operator, namespace: {{ .Release.Namespace }} } +{{- end }} diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 30c9f047c..e7a8a3fe8 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -1,3 +1,4 @@ +{{- $rbac := include "gameplane.nativeRBAC" . | fromJson -}} {{- if (eq (include "gameplane.operatorEnabled" .) "true") }} {{- $gameNamespaces := include "gameplane.operatorGameNamespaces" . | fromJsonArray }} {{- $secretNamespaces := include "gameplane.operatorSecretNamespaces" . | fromJsonArray }} @@ -47,12 +48,14 @@ rules: verbs: [get, update, patch] # Cluster is cluster-scoped; the cluster-status controller watches # registered clusters and writes their health into status. + {{- if $rbac.remoteClusters }} - apiGroups: ["gameplane.local"] resources: [clusters] verbs: [get, list, watch] - apiGroups: ["gameplane.local"] resources: [clusters/status] verbs: [get, update, patch] + {{- end }} - apiGroups: ["apps"] resources: [statefulsets, deployments] verbs: [get, list, watch] @@ -87,11 +90,6 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: [volumesnapshots, volumesnapshotcontents] verbs: [get, list, watch] - # Abort reclamation is restricted in the controller to recorded snapshot - # references and exact content UIDs; VolumeSnapshotContent is cluster-scoped. - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents] - verbs: [delete] # The GameServer controller registers Owns(&networkingv1.NetworkPolicy{}) # unconditionally (regardless of networkPolicies.enabled), and the # manager's cache is cluster-scoped (no Cache/DefaultNamespaces option @@ -165,9 +163,11 @@ rules: # gets its own Role, and pods/attach create is exec-equivalent — granting # it cluster-wide would let the operator attach to stdin of any pod in # any namespace, including kube-system and the API pod. + {{- if $rbac.operatorStopAttach }} - apiGroups: [""] resources: [pods/attach] verbs: [create] + {{- end }} # Network capture sidecar injection via ephemeral containers (reconcileCapture # in gameserver_controller.go). Namespace-scoped to this managed namespace. - apiGroups: [""] @@ -378,6 +378,9 @@ spec: - --agent-ca-secret-name={{ .Values.api.agentMTLS.caSecretRef.name }} - --game-namespaces={{ join "," $gameNamespaces }} - --control-plane-namespace={{ include "gameplane.operatorClusterCredentialsNamespace" . }} + {{- if not $rbac.remoteClusters }} + - --remote-clusters=false + {{- end }} - --agent-ca-secret-namespace={{ .Values.operator.agentCASecretNamespace | default .Release.Namespace }} {{- if and (eq (include "gameplane.apiEnabled" .) "true") (not .Values.api.standalone) }} - --module-configmap-namespace={{ include "gameplane.credentialsNamespace" . }} diff --git a/charts/gameplane/templates/validate-native-rbac.yaml b/charts/gameplane/templates/validate-native-rbac.yaml new file mode 100644 index 000000000..80bda9ec0 --- /dev/null +++ b/charts/gameplane/templates/validate-native-rbac.yaml @@ -0,0 +1,15 @@ +{{- $rbac := include "gameplane.nativeRBAC" . | fromJson -}} +{{- $managed := include "gameplane.operatorGameNamespaces" . | fromJsonArray -}} +{{- range $ns := $rbac.additionalAPIGameNamespaces -}} +{{- if not (has (include "gameplane.namespace" $ns) $managed) -}} +{{- fail "nativeRBAC.additionalAPIGameNamespaces must be a subset of the managed game namespaces" -}} +{{- end -}} +{{- end -}} +{{- if and $rbac.remoteClusters $rbac.runtimeSecretNames -}} +{{- fail "nativeRBAC.runtimeSecretNames only restricts credential reads when remoteClusters=false" -}} +{{- end -}} +{{- range $name := $rbac.runtimeSecretNames -}} +{{- if or (not (kindIs "string" $name)) (gt (len (toString $name)) 253) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" (toString $name))) -}} +{{- fail "nativeRBAC.runtimeSecretNames requires explicit Kubernetes Secret names" -}} +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 03cef2ac8..7e4b6d350 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -11,6 +11,23 @@ ## serviceMonitors — Prometheus Operator integration ## podSecurity — cluster-wide hardening knobs +# Native Kubernetes grants. The defaults retain the full native dashboard. +# See docs/native-rbac.md for single-cluster and GitOps profiles. +nativeRBAC: + catalogAdmin: true + remoteClusters: true + # With remoteClusters disabled, the API can read just these named Secrets + # for existing OIDC, notification or registry settings. + runtimeSecretNames: [] + gameSecretManagement: true + podAttach: true + # Independent of interactive dashboard access. PTY-only games need attach + # for their configured graceful-stop sequence. + operatorStopAttach: true + snapshotContentReclamation: true + # Explicit opt-in for API access to additional operator.gameNamespaces. + additionalAPIGameNamespaces: [] + image: # Component images are published to {registry}/{operator,api,agent} by the # release workflow. The release matches this default; override only to pin a diff --git a/charts/tests/check_standalone.py b/charts/tests/check_standalone.py index f0ed920c3..903462d1b 100644 --- a/charts/tests/check_standalone.py +++ b/charts/tests/check_standalone.py @@ -74,16 +74,50 @@ def check_api_availability(): render({"api": {"enabled": False, "replicas": 0}}) +def check_native_permissions(): + restricted = {"operator": {"gameNamespaces": ["tenant-b", "tenant-c"]}, + "nativeRBAC": {"catalogAdmin": False, "remoteClusters": False, "podAttach": False, + "gameSecretManagement": False, "snapshotContentReclamation": False, + "additionalAPIGameNamespaces": ["tenant-b"], "runtimeSecretNames": ["idp-a"]}} + objects = render(restricted, upgrade=True) + roles = {o["metadata"]["name"] for o in objects if o.get("kind") in ("ClusterRole", "ClusterRoleBinding")} + assert "gameplane-api-read" in roles + assert not roles.intersection({"gameplane-api-catalog-admin", "gameplane-api-cluster-registration", + "gameplane-operator-snapshot-reclaim"}) + games = [o for o in objects if o.get("kind") == "Role" and o["metadata"]["name"] == "gameplane-api-games"] + assert {o["metadata"]["namespace"] for o in games} == {"gameplane-games", "tenant-b"} + assert all(not set(rule["resources"]).intersection({"secrets", "pods/attach"}) for o in games for rule in o["rules"]) + # The API's PTY toggle leaves the operator's graceful-stop capability available. + operator = [o for o in objects if o.get("kind") == "Role" and o["metadata"]["name"] == "gameplane-operator-manage"] + assert all(any("pods/attach" in rule["resources"] for rule in o["rules"]) for o in operator) + credentials = one(objects, "Role", "gameplane-api-credentials")["rules"] + assert credentials == [ + {"apiGroups": [""], "resources": ["secrets"], "verbs": ["get"], "resourceNames": ["idp-a"]}, + {"apiGroups": [""], "resources": ["configmaps"], "verbs": ["get", "list"]}, + ] + assert all(o["metadata"]["namespace"] != "gameplane-system-credentials" for o in objects + if o.get("kind") == "Role" and o["metadata"]["name"] == "gameplane-operator-secrets") + render({"nativeRBAC": {"additionalAPIGameNamespaces": ["kube-system"]}}, failure="managed game namespaces") + render({"nativeRBAC": {"remoteClusters": "false"}}, failure="must be a boolean") + render({"nativeRBAC": {"catalogadmin": False}}, failure="unknown nativeRBAC capability") + render({"nativeRBAC": {"remoteClusters": False, "runtimeSecretNames": ["*"]}}, failure="explicit Kubernetes Secret names") + render({"nativeRBAC": {"runtimeSecretNames": ["idp-a"]}}, failure="only restricts credential reads") + + def main(): """Check deployment roles, standalone key custody, and source/archive CRDs.""" check_api_availability() + check_native_permissions() defaults = render(include_crds=True) assert_crds(defaults, EXPECTED_CRDS) one(defaults, "Deployment", "gameplane-operator") one(defaults, "Deployment", "gameplane-api") one(defaults, "ModuleSource", "default") api_rules = one(defaults, "ClusterRole", "gameplane-api-read")["rules"] - assert any(rule.get("resources") == ["clusters"] and "update" in rule["verbs"] for rule in api_rules) + assert all(set(rule["verbs"]) <= {"get", "list", "watch"} for rule in api_rules) + registration = one(defaults, "ClusterRole", "gameplane-api-cluster-registration") + assert registration["rules"] == [{"apiGroups": ["gameplane.local"], "resources": ["clusters"], + "verbs": ["create", "update", "delete"]}] assert not any("configmaps" in rule.get("resources", []) for rule in api_rules) games_role = one(defaults, "Role", "gameplane-api-games") assert games_role["metadata"]["namespace"] == "gameplane-games" diff --git a/docs/native-rbac.md b/docs/native-rbac.md new file mode 100644 index 000000000..9df5c1b68 --- /dev/null +++ b/docs/native-rbac.md @@ -0,0 +1,44 @@ +# Native Kubernetes permissions + +Helm preserves the full native dashboard's existing access on upgrade. Use the following values for a single-cluster installation where GitOps owns catalogs, runtime credentials and backup destinations. Add each API-managed game namespace to both lists. The API retains writes in its primary games namespace. Operator-managed namespaces receive additional API writes only when explicitly listed. + +```yaml +operator: + gameNamespaces: [tenant-b] +nativeRBAC: + catalogAdmin: false + remoteClusters: false + runtimeSecretNames: [oidc-production] + gameSecretManagement: false + podAttach: false + operatorStopAttach: true + snapshotContentReclamation: false + additionalAPIGameNamespaces: [tenant-b] +``` + +| Kubernetes identity | Resources and allowed actions | Boundary | +| --- | --- | --- | +| Native API, `-api-read` | Read the catalog, Gameplane resources and cluster inventory. List pods for node usage. Remote mode also permits Cluster discovery. | Read-only. It has no Secret, pod-attach or catalog-write grant. | +| Native API, `-api-catalog-admin` | Create, update, patch and delete cluster-wide templates, modules and module sources when `catalogAdmin` is enabled. The API can also manage module-upload ConfigMaps in its credential namespace. | In GitOps mode the API can read the catalog and ConfigMaps. The operator continues to reconcile administrator-provisioned module resources. | +| Native API, `-api-cluster-registration` and `-api-credentials` | Remote mode grants registration writes plus get/create/update/delete on Secrets in the dedicated credentials namespace. This includes the managed Secret routes for OIDC, notifications and registry credentials. | Local-only mode omits registration, disables the remote watcher and grants get only on explicitly named runtime Secrets. Supply only the OIDC, notification or registry names the API needs. Kubernetes RBAC cannot distinguish kubeconfigs from other payloads in an explicitly allowed Secret. | +| Native API, `-api-games` | Manage game resources and read pod logs in the primary and explicitly authorized extra game namespaces. `gameSecretManagement` adds Secret CRUD for backup destinations and tunnel credentials. `podAttach` enables the dashboard PTY. | These options grant namespace-wide Secret or attach access to the API process. Use dedicated game namespaces. The in-process server ownership checks still apply to user requests. | +| Operator | Owns game workload resources, services, backup jobs and per-server roles in each managed namespace. Reads the configured agent signing Secret and issues agent certificates. `operatorStopAttach` permits the PTY stop sequence independently of dashboard PTY permissions. | Game agents, wakers and the native API cannot read the issuer key. Keep operator attach enabled for modules that use PTY graceful shutdown. The operator drops the remote status watcher and its dedicated kubeconfig namespace from the Secret cache in local-only mode. | +| Operator, `-operator-snapshot-reclaim` | When enabled, deletes cluster-scoped VolumeSnapshotContent on an aborted CSI backup after verifying owner, snapshot reference, deletion policy and UID. | With this grant disabled, the operator can still create and read namespaced snapshots. If content survives, read `SnapshotCleaned`, verify the recorded UIDs and reclaim the orphan as storage administrator. The development operator manifest also leaves cluster-wide content deletion to the administrator. | +| Per-server `-agent` | Get/patch the named GameServer's status. | Cannot patch the peer's status, modify game specs or read Secrets. | +| Per-server `-waker-sa` | Get/patch its named GameServer. Wake admission further constrains modifications. | Cannot patch a peer server or status and cannot read Secrets. | +| Standalone panel | Uses SQL-backed registration and encrypted credentials. The standalone API Pod has ServiceAccount token automount disabled. | Helm creates no native Kubernetes RBAC for this identity. Permissions in registered remote clusters come from the supplied kubeconfig. | + +Cluster bootstrap tokens and short-lived Kubernetes user certificates have their own existing `clusterOps.enabled` switch. Leave it disabled unless a trusted panel must perform those actions. Its signer permission is separate from the operator's agent mTLS CA. + +## Upgrade and verification + +The chart retains the existing `-api-read` ClusterRoleBinding and its immutable `roleRef`. Helm removes write verbs from the existing ClusterRole and creates separately named roles and bindings for enabled administrative capabilities. On disabling a capability, Helm removes its dedicated binding or the verbs on its namespaced Role. Check other bindings installed outside the release, since Kubernetes combines their permissions. + +```sh +kubectl auth can-i get gametemplates.gameplane.local --as=system:serviceaccount:gameplane-system:gameplane-api +kubectl auth can-i patch gametemplates.gameplane.local --as=system:serviceaccount:gameplane-system:gameplane-api +kubectl auth can-i get secret/gameplane-agent-ca -n gameplane-system --as=system:serviceaccount:gameplane-system:gameplane-api +kubectl auth can-i delete volumesnapshotcontents.snapshot.storage.k8s.io --as=system:serviceaccount:gameplane-system:gameplane-operator +``` + +The Kind upgrade job compares the read binding's UID and effective catalog mutation before and after a published upgrade. It then exercises the full and restricted profiles, both named servers, selected namespaces, server-side catalog and CSI content dry runs, and a separate standalone Helm release. The restricted-workloads job exercises issuer denial and the real rotating agent token. The Helm installation suite covers mTLS client reissuance on CA rotation. diff --git a/docs/security.md b/docs/security.md index b316664c9..4676a769d 100644 --- a/docs/security.md +++ b/docs/security.md @@ -985,6 +985,8 @@ install is locked out. See ## API cluster-wide pod list +The [native Kubernetes permission matrix](native-rbac.md) describes catalog administration, remote kubeconfig custody, game Secret and PTY access, and CSI content reclamation. The chart binds these capabilities according to the selected installation profile. + The API's `-api-read` ClusterRole grants `list` on Pods in **every** namespace, so the Cluster page can show per-node pod usage (`countNodePods` in `api/internal/handlers/cluster.go`). A pod list returns diff --git a/operator/cmd/main.go b/operator/cmd/main.go index ccd430fb6..00d05fbd1 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -141,6 +141,7 @@ func main() { moduleConfigMapNamespace string moduleLocalRoot string controlPlaneNamespace string + remoteClusters bool gameNamespaces string addressManager string metalLBNamespace string @@ -263,6 +264,7 @@ func main() { } flag.StringVar(&controlPlaneNamespace, "control-plane-namespace", controlPlaneNamespaceDefault, "Namespace where registered cluster kubeconfig Secrets are stored.") + flag.BoolVar(&remoteClusters, "remote-clusters", true, "watch registered clusters and their kubeconfig credentials") addressManagerDefault := os.Getenv("GAMEPLANE_ADDRESS_MANAGER") if addressManagerDefault == "" { addressManagerDefault = "none" @@ -440,7 +442,11 @@ func main() { probeIngressFromCIDR[i] = ipnet.String() } - secretCache, err := secretscope.CacheOptions(gameNamespaces, controlPlaneNamespace, agentCASecretNamespace, moduleNamespace) + secretNamespaces := []string{agentCASecretNamespace, moduleNamespace} + if remoteClusters { + secretNamespaces = append(secretNamespaces, controlPlaneNamespace) + } + secretCache, err := secretscope.CacheOptions(gameNamespaces, secretNamespaces...) if err != nil { setupLog.Error(err, "invalid Secret namespace configuration") os.Exit(1) @@ -608,13 +614,15 @@ func main() { setupLog.Error(err, "unable to set up controller", "controller", "Module") os.Exit(1) } - if err := (&controller.ClusterStatusReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Namespace: controlPlaneNamespace, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to set up controller", "controller", "Cluster") - os.Exit(1) + if remoteClusters { + if err := (&controller.ClusterStatusReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Namespace: controlPlaneNamespace, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to set up controller", "controller", "Cluster") + os.Exit(1) + } } // Fleet metrics: report how many GameServers and Backups sit in each phase, diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index ad3a3ce31..66736af00 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -157,14 +157,6 @@ rules: - snapshot.storage.k8s.io resources: - volumesnapshotcontents - verbs: - - delete - - get - - list - - watch -- apiGroups: - - snapshot.storage.k8s.io - resources: - volumesnapshots verbs: - get diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index 694429e53..62009db91 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -239,7 +239,8 @@ func resticImageOrDefault(image string) string { // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch // +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;list;watch -// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch;delete +// Optional cluster-wide content deletion is bound separately by the deployment. +// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { return requeueOnConflict(r.reconcile(ctx, req)) diff --git a/operator/internal/secretscope/cache.go b/operator/internal/secretscope/cache.go index b51be4af8..2dd57a49d 100644 --- a/operator/internal/secretscope/cache.go +++ b/operator/internal/secretscope/cache.go @@ -15,9 +15,9 @@ import ( // without restricting cluster-scoped Gameplane resources or other informers. // Empty names are rejected: controller-runtime treats an empty namespace as // all namespaces, and an empty ByObject namespace map inherits the defaults. -func CacheOptions(games, controlPlane, ca, modules string) (cache.Options, error) { +func CacheOptions(games string, secretNamespaces ...string) (cache.Options, error) { namespaces := make(map[string]cache.Config) - for _, namespace := range append(strings.Split(games, ","), controlPlane, ca, modules) { + for _, namespace := range append(strings.Split(games, ","), secretNamespaces...) { if problems := validation.IsDNS1123Label(namespace); len(problems) != 0 { return cache.Options{}, fmt.Errorf("secret namespace %q must be an explicit DNS-label namespace: %s", namespace, strings.Join(problems, "; ")) } diff --git a/operator/specs.md b/operator/specs.md index 2fa58529d..937fb95c3 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -436,7 +436,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal CSI observation waits for ready/error, but cancellation durably invalidates incomplete captures before save-on. Force-deleted Backup recovery marks the exact owned snapshot aborted and requests UID-bound deletion; a missing Backup cannot expose that snapshot as a valid Gameplane capture. - - CSI capture has a persisted deadline before quiesce, default 10m (`--backup-csi-capture-timeout`, Helm `operator.backupCSICaptureTimeout`), independent of Restic. API/creation errors, deadline expiry and deletion persist `backup.gameplane.local/snapshot-aborted` as a terminal decision, clear snapshot identity and fail the Backup before save-on. Late readiness cannot revive it. Snapshot/content identity is retained separately for reclamation. Cleanup verifies snapshot ownership and recorded UID, content reference UID/name/namespace and content UID, and deletes with UID preconditions. `Retain` or unverified resources remain for administrator action with names/UIDs in `SnapshotCleaned` conditions/events. Cleanup errors retry independently of save-on; deletion retains the Backup finalizer for pending reclamation but does not retain the game owner after confirmed save-on. Cluster RBAC adds content delete for verified `Delete`-policy resources. + - CSI capture has a persisted deadline before quiesce, default 10m (`--backup-csi-capture-timeout`, Helm `operator.backupCSICaptureTimeout`), independent of Restic. API/creation errors, deadline expiry and deletion persist `backup.gameplane.local/snapshot-aborted` as a terminal decision, clear snapshot identity and fail the Backup before save-on. Late readiness cannot revive it. Snapshot/content identity is retained separately for reclamation. Cleanup verifies snapshot ownership and recorded UID, content reference UID/name/namespace and content UID, and deletes with UID preconditions. `Retain` or unverified resources remain for administrator action with names/UIDs in `SnapshotCleaned` conditions/events. Cleanup errors retry independently of save-on; deletion retains the Backup finalizer for pending reclamation but does not retain the game owner after confirmed save-on. The optional Helm `nativeRBAC.snapshotContentReclamation` grant authorizes content deletion for verified `Delete`-policy resources. Without it, a surviving content object requires administrator reclamation. See the [native Kubernetes permission matrix](../docs/native-rbac.md). - New PVCs referencing a Gameplane Backup-owned VolumeSnapshot require uncached successful-owner/snapshotID/UID/readiness checks and reject abort markers, including raw GameServer dataSource references. Existing PVCs do not depend on retained source metadata. External snapshots without a Gameplane Backup owner preserve the administrator direct-CRD contract; API authorization is separate. - Restic Job: mount data PVC, run restic backup to configured destination (S3, B2, rest-server, local fs). Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); only a Failed=True Job condition (set by jobPermanentlyFailed check) marks the Backup Failed. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - Snapshot tracking: parse restic JSON output from container logs to extract snapshot ID. diff --git a/test/e2e/game_admission_e2e_test.go b/test/e2e/game_admission_e2e_test.go index 4d3340327..80917ff64 100644 --- a/test/e2e/game_admission_e2e_test.go +++ b/test/e2e/game_admission_e2e_test.go @@ -54,9 +54,7 @@ func checkGameAdmission(t *testing.T, extraNS string, game *corev1.Pod) { core := envInstance.K8s.CoreV1() dryRun := metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}} api := gameplaneServiceAccountClient(t, "gameplane-api") - if _, err := api.CoreV1().Secrets("gameplane-system").Get(ctx, "gameplane-agent-ca", metav1.GetOptions{}); !apierrors.IsForbidden(err) { - t.Fatalf("API reading agent signer: %v", err) - } + verifyAPIIssuerDenied(t, api) checkModuleConfigMapIsolation(t, api) for _, ns := range []string{extraNS, game.Namespace, "gameplane-system", credentialsNS} { secrets := core.Secrets(ns) @@ -152,6 +150,13 @@ func checkModuleConfigMapIsolation(t *testing.T, api kubernetes.Interface) { } } +func verifyAPIIssuerDenied(t *testing.T, api kubernetes.Interface) { + t.Helper() + if _, err := api.CoreV1().Secrets("gameplane-system").Get(t.Context(), "gameplane-agent-ca", metav1.GetOptions{}); !apierrors.IsForbidden(err) { + t.Fatalf("API reading agent signer: %v", err) + } +} + // gameplaneServiceAccountClient includes the groups Kubernetes assigns on token // authentication so group-based RBAC applies to the impersonated request. func gameplaneServiceAccountClient(t *testing.T, name string) kubernetes.Interface { diff --git a/test/e2e/native_rbac_e2e_test.go b/test/e2e/native_rbac_e2e_test.go new file mode 100644 index 000000000..5028be913 --- /dev/null +++ b/test/e2e/native_rbac_e2e_test.go @@ -0,0 +1,235 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const panelNamespace = "gameplane-system" + +func verifyLocalClusterDiscovery(t *testing.T, client *APIClient) { + t.Helper() + resp, body, err := client.Get("/clusters") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var result struct{ Items []struct{ Name string } } + if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &result) != nil || + len(result.Items) != 1 || result.Items[0].Name != "local" { + t.Fatalf("local cluster discovery returned %d: %s", resp.StatusCode, body) + } +} + +func nativeReadBinding(t *testing.T) types.UID { + t.Helper() + binding, err := envInstance.K8s.RbacV1().ClusterRoleBindings().Get(t.Context(), "gameplane-api-read", metav1.GetOptions{}) + if err != nil || binding.RoleRef.Kind != "ClusterRole" || binding.RoleRef.Name != "gameplane-api-read" || binding.UID == "" { + t.Fatalf("native API read binding: %v, %v", binding, err) + } + return binding.UID +} + +func checkNativeCatalogAccess(t *testing.T, template string, allow bool) { + t.Helper() + out, err := envInstance.Kubectl(t.Context(), "--as=system:serviceaccount:gameplane-system:gameplane-api", + "annotate", "gametemplate", template, "e2e-rbac=verified", "--overwrite", "--dry-run=server") + if (allow && err != nil) || (!allow && (err == nil || !strings.Contains(out, "Forbidden"))) { + t.Fatalf("catalog mutation allow=%t: %v: %s", allow, err, out) + } +} + +type nativeGrant struct { + actor, verb, group, resource, subresource, namespace, name string + allow bool +} + +func verifyNativeGrants(t *testing.T, grants []nativeGrant) { + t.Helper() + for _, g := range grants { + ns, sa, ok := strings.Cut(g.actor, "/") + if !ok { + t.Fatalf("invalid service account %q", g.actor) + } + sar := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ + User: "system:serviceaccount:" + ns + ":" + sa, + Groups: []string{"system:authenticated", "system:serviceaccounts", "system:serviceaccounts:" + ns}, + ResourceAttributes: &authorizationv1.ResourceAttributes{Namespace: g.namespace, Verb: g.verb, + Group: g.group, Resource: g.resource, Subresource: g.subresource, Name: g.name}, + }} + // Bindings are applied through a different apiserver watch from the + // Deployment rollout. Tolerate authorizer propagation on transitions. + envInstance.Eventually(t, 30*time.Second, func() (bool, string) { + verdict, err := envInstance.K8s.AuthorizationV1().SubjectAccessReviews().Create(t.Context(), sar, metav1.CreateOptions{}) + if err != nil { + return false, err.Error() + } + return verdict.Status.EvaluationError == "" && verdict.Status.Allowed == g.allow, + fmt.Sprintf("%+v: %+v", g, verdict.Status) + }) + } +} + +func checkNativeRBACUpgrade(t *testing.T, games, game, template string, legacyReadBinding types.UID) { + t.Helper() + if nativeReadBinding(t) != legacyReadBinding { + t.Fatal("upgrade replaced the legacy read binding") + } + checkNativeCatalogAccess(t, template, true) + const api = panelNamespace + "/gameplane-api" + const operator = panelNamespace + "/gameplane-operator" + const plane = "gameplane.local" + const snapshots = "snapshot.storage.k8s.io" + peer := game + "-peer" + applyBusyboxGameServer(t, games, peer, template) + waitGameContainerReady(t, games, peer+"-0", 3*time.Minute) + agent, waker := games+"/"+game+"-agent", games+"/"+game+"-waker-sa" + peerAgent, peerWaker := games+"/"+peer+"-agent", games+"/"+peer+"-waker-sa" + extra := "e2e-rbac-extra" + _, err := envInstance.K8s.CoreV1().Namespaces().Create(t.Context(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: extra, Labels: map[string]string{"pod-security.kubernetes.io/enforce": "restricted"}, + }}, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), extra, metav1.DeleteOptions{}) + }) + checkSnapshotContentGrant := prepareSnapshotContentCheck(t, games, game) + for _, full := range []bool{true, false} { + if !full { + helmUpgradeReuse(t.Context(), t, "nativeRBAC.remoteClusters=false", "nativeRBAC.catalogAdmin=false", + "nativeRBAC.podAttach=false", "nativeRBAC.operatorStopAttach=false", "nativeRBAC.gameSecretManagement=false", + "nativeRBAC.snapshotContentReclamation=false", "nativeRBAC.runtimeSecretNames[0]=idp-a", + "operator.gameNamespaces[0]="+extra, "nativeRBAC.additionalAPIGameNamespaces[0]="+extra) + } + verifyNativeGrants(t, []nativeGrant{ + {api, "list", plane, "gametemplates", "", "", "", true}, + {api, "patch", plane, "gametemplates", "", "", template, full}, + {api, "create", plane, "modules", "", "", "", full}, + {api, "create", plane, "clusters", "", "", "", full}, + {api, "list", plane, "clusters", "", "", "", full}, + {api, "get", "", "secrets", "", credentialsNS, "cluster-canary-kubeconfig", full}, + {api, "get", "", "secrets", "", credentialsNS, "idp-a", true}, + {api, "list", "", "secrets", "", credentialsNS, "", false}, + {api, "update", "", "secrets", "", credentialsNS, "idp-a", full}, + {api, "get", "", "secrets", "", panelNamespace, "gameplane-agent-ca", false}, + {api, "list", "", "secrets", "", games, "", full}, + {api, "get", "", "pods", "log", games, game + "-0", true}, + {api, "create", "", "pods", "attach", games, game + "-0", full}, + {api, "create", plane, "gameservers", "", extra, "", !full}, + {api, "create", plane, "gameservers", "", "default", "", false}, + {api, "approve", "certificates.k8s.io", "signers", "", "", "kubernetes.io/kube-apiserver-client", false}, + {operator, "get", "", "secrets", "", panelNamespace, "gameplane-agent-ca", true}, + {operator, "get", "", "secrets", "", credentialsNS, "cluster-canary-kubeconfig", full}, + {operator, "get", "", "secrets", "", "default", "unrelated", false}, + {operator, "delete", snapshots, "volumesnapshotcontents", "", "", "rbac-content", full}, + {operator, "create", snapshots, "volumesnapshots", "", games, "", true}, + {operator, "create", "", "pods", "attach", games, game + "-0", full}, + {agent, "patch", plane, "gameservers", "status", games, game, true}, + {agent, "patch", plane, "gameservers", "status", games, peer, false}, + {agent, "patch", plane, "gameservers", "status", extra, game, false}, + {agent, "get", "", "secrets", "", panelNamespace, "gameplane-agent-ca", false}, + {peerAgent, "patch", plane, "gameservers", "status", games, peer, true}, + {waker, "patch", plane, "gameservers", "", games, game, true}, + {waker, "patch", plane, "gameservers", "", games, peer, false}, + {waker, "patch", plane, "gameservers", "status", games, game, false}, + {peerWaker, "patch", plane, "gameservers", "", games, peer, true}, + }) + verifyAPIIssuerDenied(t, gameplaneServiceAccountClient(t, "gameplane-api")) + checkNativeCatalogAccess(t, template, full) + checkSnapshotContentGrant(full) + } + checkStandaloneNativeGrants(t, games, game) + // The enclosing upgrade fixture uninstalls the operator after this + // check. Let it finish the peer's finalizer while it can still reconcile. + servers := envInstance.Dyn.Resource(gameServerGVR).Namespace(games) + if err := servers.Delete(t.Context(), peer, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + _, err := servers.Get(t.Context(), peer, metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprintf("waiting for peer finalizer: %v", err) + }) +} + +// Install the same CSI API definitions as the controller's envtest. A real +// apiserver authorizes the operator's deletion request. A server-side dry run +// avoids deleting or provisioning any underlying storage. +func prepareSnapshotContentCheck(t *testing.T, ns, game string) func(bool) { + t.Helper() + crds := filepath.Join("..", "..", "operator", "test", "crds", "snapshot") + if out, err := envInstance.Kubectl(t.Context(), "apply", "--server-side", "-f", crds); err != nil { + t.Fatalf("install CSI CRDs: %v: %s", err, out) + } + if out, err := envInstance.Kubectl(t.Context(), "wait", "--for=condition=Established", + "crd/volumesnapshotcontents.snapshot.storage.k8s.io", "--timeout=1m"); err != nil { + t.Fatalf("wait for snapshot content API: %v: %s", err, out) + } + const content = "e2e-gameplane-rbac-content" + obj := fmt.Sprintf(`{"apiVersion":"snapshot.storage.k8s.io/v1","kind":"VolumeSnapshotContent","metadata":{"name":%q},"spec":{"deletionPolicy":"Retain","driver":"e2e.gameplane.local","source":{"snapshotHandle":"e2e-only"},"volumeSnapshotRef":{"name":%q,"namespace":%q}}}`, content, game, ns) + if out, err := envInstance.KubectlWithStdin(t.Context(), obj, "create", "-f", "-"); err != nil { + t.Fatalf("create snapshot content: %v: %s", err, out) + } + t.Cleanup(func() { + _, _ = envInstance.Kubectl(context.Background(), "delete", "volumesnapshotcontent", content, "--ignore-not-found", "--wait=false") + }) + return func(allow bool) { + t.Helper() + out, err := envInstance.Kubectl(t.Context(), "--as=system:serviceaccount:gameplane-system:gameplane-operator", + "delete", "volumesnapshotcontent", content, "--dry-run=server") + if (allow && err != nil) || (!allow && (err == nil || !strings.Contains(out, "Forbidden"))) { + t.Fatalf("operator snapshot content deletion allow=%t: %v: %s", allow, err, out) + } + } +} + +func checkStandaloneNativeGrants(t *testing.T, games, game string) { + t.Helper() + const ns = "e2e-rbac-standalone" + tag := envInstance.Tag + if tag == "" { + tag = "e2e" + } + chart := filepath.Join("..", "..", "charts", "gameplane") + cmd := exec.CommandContext(t.Context(), "helm", "install", "rbac-panel", chart, "--namespace", ns, + "--kube-context", envInstance.Context, "--create-namespace", + "--set", "image.registry=gameplane-test,image.tag="+tag+",operator.enabled=false,api.standalone=true,web.enabled=false,ingress.enabled=false", + "--wait", "--timeout", "4m") + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) + }) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("install standalone panel: %v: %s", err, out) + } + deployment, err := envInstance.K8s.AppsV1().Deployments(ns).Get(t.Context(), "gameplane-api", metav1.GetOptions{}) + if err != nil || deployment.Spec.Template.Spec.ServiceAccountName != "gameplane-api" || + deployment.Spec.Template.Spec.AutomountServiceAccountToken == nil || *deployment.Spec.Template.Spec.AutomountServiceAccountToken { + t.Fatalf("standalone panel unexpectedly uses local API credentials: %v", err) + } + actor := ns + "/gameplane-api" + verifyNativeGrants(t, []nativeGrant{ + {actor, "list", "gameplane.local", "clusters", "", "", "", false}, + {actor, "create", "gameplane.local", "gametemplates", "", "", "", false}, + {actor, "get", "", "secrets", "", panelNamespace, "gameplane-agent-ca", false}, + {actor, "get", "", "pods", "attach", games, game + "-0", false}, + }) + if _, err := envInstance.K8s.RbacV1().ClusterRoleBindings().Get(t.Context(), "rbac-panel-api-read", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("standalone panel has an unexpected native API binding: %v", err) + } +} diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 0453f8fbf..263f98c04 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -74,6 +74,8 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { applyBusyboxGameServer(t, ns, gs, tmpl) waitPVCBound(t, ns, gs+"-data", 90*time.Second) waitGameContainerReady(t, ns, gs+"-0", 2*time.Minute) + legacyReadBinding := nativeReadBinding(t) + checkNativeCatalogAccess(t, tmpl, true) // `sync` so the bytes are on the PVC's backing store before the pod is // restarted by the upgrade, not sitting in the page cache. @@ -270,6 +272,12 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { t.Errorf("telemetry sendMetrics is true after an upgrade with no saved choice; body=%s", body) } + checkNativeRBACUpgrade(t, ns, gs, tmpl, legacyReadBinding) + localClient := envInstance.APIClient(t, adminUser, adminPass) + defer localClient.Close() + waitAgentReachable(t, localClient, gs) + verifyLocalClusterDiscovery(t, localClient) + // ---- 7. F-218: a fresh `helm install` over leftover, STALE CRDs ------ // // Deliberately LAST, after steps 3-6 have proven the pre-upgrade hook From 36ed48170ed1b2ff79b081d568b39f768d0de106 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 23:51:59 -0500 Subject: [PATCH 2/5] fix(api): preserve the native cluster route ownership check Signed-off-by: Baylem --- api/cmd/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/cmd/main.go b/api/cmd/main.go index 532b4b876..afae1ac68 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -396,11 +396,11 @@ func main() { handlers.MountAuthProviderSecrets(p, k8s, credentialsNS) handlers.MountCluster(p, reg, store, Version, cfg.clusterOps, cfg.updateChannel) handlers.MountClusterActions(p, k8s, cfg.clusterOps, cfg.clusterExternalAddress) - clusterManagement := k8s - if !cfg.remoteClusters { - clusterManagement = nil + if cfg.remoteClusters { + handlers.MountClusters(p, reg, k8s, credentialsNS) + } else { + handlers.MountClusters(p, reg, nil, credentialsNS) } - handlers.MountClusters(p, reg, clusterManagement, credentialsNS) handlers.MountInstallation(p, cfg.standalone, Version) handlers.MountEvents(p, reg) handlers.MountDestinations(p, reg) From 3c7a4e48ca2aff3c04f56662a733d2a87be398eb Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 00:18:20 -0500 Subject: [PATCH 3/5] test(rbac): verify local-only discovery and remote route exclusion Signed-off-by: Baylem --- .../handlers/clusters_discovery_test.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/api/internal/handlers/clusters_discovery_test.go b/api/internal/handlers/clusters_discovery_test.go index 867486949..01af2d5dd 100644 --- a/api/internal/handlers/clusters_discovery_test.go +++ b/api/internal/handlers/clusters_discovery_test.go @@ -79,6 +79,37 @@ func TestClusterDiscovery_FiltersRegistrationsAndInventoryCapability(t *testing. } } +func TestClusterDiscovery_LocalOnlyRoutes(t *testing.T) { + home := fakeKubeClientWithClusters(newCluster("remote", nil, nil)) + reg := clusterTestRegistry(home) + router := chi.NewRouter() + MountClusters(router, reg, nil, "gameplane-system") + user := inventoryUser("*", "*", "cluster:read") + + list := inventoryRequest(t, router, user, http.MethodGet, "/clusters/") + var got clustersListResp + if err := json.Unmarshal(list.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if list.Code != http.StatusOK || len(got.Items) != 1 || got.Items[0].Name != "local" || !got.Items[0].CanViewInventory { + t.Fatalf("local-only discovery: status=%d items=%+v", list.Code, got.Items) + } + if kubeClientCalls(t, home) != 0 { + t.Fatal("local-only discovery contacted Kubernetes for remote registrations") + } + for _, route := range []struct{ method, path string }{ + {http.MethodPost, "/clusters/"}, + {http.MethodDelete, "/clusters/remote"}, + {http.MethodPut, "/clusters/remote/kubeconfig"}, + {http.MethodPut, "/clusters/remote/gateway"}, + {http.MethodDelete, "/clusters/remote/gateway"}, + } { + if router.Match(chi.NewRouteContext(), route.method, route.path) { + t.Errorf("remote administration route mounted: %s %s", route.method, route.path) + } + } +} + func TestClusterDiscovery_RetainsDisconnectedRegistration(t *testing.T) { home := fakeKubeClientWithClusters(newCluster("offline", map[string]any{"displayName": "Offline cluster"}, map[string]any{"phase": "Healthy"})) reg := clusterTestRegistry(home) From fbb29d58cb9c7fb8b050e56e7e3c6dce1aba2aea Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 00:57:01 -0500 Subject: [PATCH 4/5] test(api): exercise local-only routes over HTTP Signed-off-by: Baylem --- .../handlers/clusters_discovery_test.go | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/api/internal/handlers/clusters_discovery_test.go b/api/internal/handlers/clusters_discovery_test.go index 01af2d5dd..59af723d8 100644 --- a/api/internal/handlers/clusters_discovery_test.go +++ b/api/internal/handlers/clusters_discovery_test.go @@ -97,17 +97,24 @@ func TestClusterDiscovery_LocalOnlyRoutes(t *testing.T) { if kubeClientCalls(t, home) != 0 { t.Fatal("local-only discovery contacted Kubernetes for remote registrations") } - for _, route := range []struct{ method, path string }{ - {http.MethodPost, "/clusters/"}, - {http.MethodDelete, "/clusters/remote"}, - {http.MethodPut, "/clusters/remote/kubeconfig"}, - {http.MethodPut, "/clusters/remote/gateway"}, - {http.MethodDelete, "/clusters/remote/gateway"}, + for _, route := range []struct { + method, path string + status int + }{ + {http.MethodPost, "/clusters/", http.StatusMethodNotAllowed}, + {http.MethodDelete, "/clusters/remote", http.StatusNotFound}, + {http.MethodPut, "/clusters/remote/kubeconfig", http.StatusNotFound}, + {http.MethodPut, "/clusters/remote/gateway", http.StatusNotFound}, + {http.MethodDelete, "/clusters/remote/gateway", http.StatusNotFound}, } { - if router.Match(chi.NewRouteContext(), route.method, route.path) { - t.Errorf("remote administration route mounted: %s %s", route.method, route.path) + response := inventoryRequest(t, router, user, route.method, route.path) + if response.Code != route.status { + t.Errorf("%s %s = %d, want %d", route.method, route.path, response.Code, route.status) } } + if kubeClientCalls(t, home) != 0 { + t.Fatal("local-only administration request contacted Kubernetes") + } } func TestClusterDiscovery_RetainsDisconnectedRegistration(t *testing.T) { From cb71cb836be4f9c3bfd7725d32a96d78de9fd599 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 02:37:17 -0500 Subject: [PATCH 5/5] fix(e2e): close single-request webhook connections explicitly Signed-off-by: Baylem --- test/e2e/fixtures/notify-sink.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/fixtures/notify-sink.yaml b/test/e2e/fixtures/notify-sink.yaml index ed4df0162..1ffec1523 100644 --- a/test/e2e/fixtures/notify-sink.yaml +++ b/test/e2e/fixtures/notify-sink.yaml @@ -16,7 +16,9 @@ data: body=$(head -c "${CONTENT_LENGTH:-0}") printf '%s\t%s\n' "$QUERY_STRING" "$body" >> /tmp/hooks.log printf '%s\t%s\tbody-read\n' "$(date -u +%FT%TZ)" "$QUERY_STRING" >> /tmp/requests.log - printf 'Status: 204 No Content\r\n\r\n' + # BusyBox closes each CGI connection. Tell HTTP/1.1 clients before + # they reuse this socket for the next notification. + printf 'Status: 204 No Content\r\nConnection: close\r\n\r\n' printf '%s\t%s\tresponse-written\n' "$(date -u +%FT%TZ)" "$QUERY_STRING" >> /tmp/requests.log --- apiVersion: apps/v1