From cd7583e808e83cfffd7c335032b011fcb963ab8b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 20:09:08 +0000 Subject: [PATCH 1/2] fix(chart): allow detected apiserver endpoints in agent egress policy On k3s and kubeadm single-node installs the kubernetes Service is DNATed to the node's own IP. When that IP is public, the default allow-agent-to-apiserver policy (RFC1918 + link-local only) blocks it, every agent heartbeat times out and servers stay "Starting". When networkPolicies.apiServerCIDRs is unset, the chart now also allows the kubernetes Service ClusterIPs and EndpointSlice addresses it looks up at install/upgrade time (as /32 or /128). Explicit lists and offline renders are unchanged. Applies to the sentinel's apiserver policy too. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Claude --- CHANGELOG.md | 9 ++++ charts/gameplane/templates/_helpers.tpl | 44 +++++++++++++++++++ .../gameplane/templates/networkpolicies.yaml | 31 +++---------- charts/gameplane/values.yaml | 8 +++- docs/install.md | 2 +- docs/security.md | 5 ++- 6 files changed, 69 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfdc17b9..69caf3211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ reaches `1.0.0`. Pre-1.0 minor versions may contain breaking changes. ## [Unreleased] +### Fixed + +- **Servers stuck in "Starting" when the API server has a public address:** + the chart's `allow-agent-to-apiserver` and `allow-sentinel-to-apiserver` + policies only allowed private ranges by default, so on a k3s node with a + public IP every agent heartbeat timed out. When + `networkPolicies.apiServerCIDRs` is unset, the chart now also allows the + `kubernetes` Service endpoints it detects at install or upgrade time. + ## [0.4.0] — 2026-11-10 A pre-v1 release with every change made since v0.3.0: an independent central diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index 44bf31ee3..bb80d1b29 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -189,3 +189,47 @@ mergeOverwrite preserves explicit zero/empty values so validation can reject the {{- end -}} {{- uniq $namespaces | toJson -}} {{- end -}} + +{{- /* +gameplane.apiServerEgressPeers renders the NetworkPolicy `to:` peers of the +agent and sentinel apiserver egress rules. An explicit +networkPolicies.apiServerCIDRs list wins. Otherwise it renders the RFC1918 + +link-local fallback plus every address the live cluster reports for the +`kubernetes` Service: its ClusterIPs and its EndpointSlice endpoint addresses. +On k3s and kubeadm single-node installs that endpoint is the node's own IP, +which is often public and outside the fallback; without it heartbeats time +out and every server stays "Starting". Offline renders (helm template, client +dry-run) get empty lookups and keep the plain fallback. Addresses are read at +install/upgrade time, so re-run helm upgrade if the API endpoint moves. +*/}} +{{- define "gameplane.apiServerEgressPeers" -}} +{{- $cidrs := .Values.networkPolicies.apiServerCIDRs | default (list) -}} +{{- if not $cidrs -}} +{{- $cidrs = list "10.0.0.0/8" "172.16.0.0/12" "192.168.0.0/16" "169.254.0.0/16" -}} +{{- $addrs := list -}} +{{- $svc := lookup "v1" "Service" "default" "kubernetes" -}} +{{- range (dig "spec" "clusterIPs" (list) $svc) -}} +{{- $addrs = append $addrs . -}} +{{- end -}} +{{- $slices := lookup "discovery.k8s.io/v1" "EndpointSlice" "default" "" -}} +{{- range (dig "items" (list) $slices) -}} +{{- if eq (dig "metadata" "labels" "kubernetes.io/service-name" "" .) "kubernetes" -}} +{{- range (dig "endpoints" (list) .) -}} +{{- range (dig "addresses" (list) .) -}} +{{- $addrs = append $addrs . -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range $addrs -}} +{{- if contains ":" . -}} +{{- $cidrs = append $cidrs (printf "%s/128" .) -}} +{{- else -}} +{{- $cidrs = append $cidrs (printf "%s/32" .) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range (uniq $cidrs) }} +- ipBlock: { cidr: {{ . | quote }} } +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index ed2ae0a8a..3d8942297 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -40,9 +40,10 @@ spec: # Allow the agent sidecar to reach the kube-apiserver for its status # heartbeat (gameservers/status patches). Service VIP traffic is DNATed # to the apiserver endpoint before policy evaluation on most CNIs, so an -# ipBlock toward the endpoint address is required; like the kubelet -# probes policy below, default to RFC1918 + link-local and let operators -# narrow it via networkPolicies.apiServerCIDRs. +# ipBlock toward the endpoint address is required. Unset +# networkPolicies.apiServerCIDRs means RFC1918 + link-local plus the +# kubernetes Service endpoints detected at install time (a k3s node's +# public IP, for example); set the list to narrow it. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -56,17 +57,7 @@ spec: policyTypes: [Egress] egress: - to: - {{- $apiCidrs := (.Values.networkPolicies.apiServerCIDRs | default (list)) }} - {{- if $apiCidrs }} - {{- range $apiCidrs }} - - ipBlock: { cidr: {{ . | quote }} } - {{- end }} - {{- else }} - - ipBlock: { cidr: 10.0.0.0/8 } - - ipBlock: { cidr: 172.16.0.0/12 } - - ipBlock: { cidr: 192.168.0.0/16 } - - ipBlock: { cidr: 169.254.0.0/16 } - {{- end }} + {{- include "gameplane.apiServerEgressPeers" . | trim | nindent 8 }} ports: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } @@ -341,17 +332,7 @@ spec: policyTypes: [Egress] egress: - to: - {{- $apiCidrs := (.Values.networkPolicies.apiServerCIDRs | default (list)) }} - {{- if $apiCidrs }} - {{- range $apiCidrs }} - - ipBlock: { cidr: {{ . | quote }} } - {{- end }} - {{- else }} - - ipBlock: { cidr: 10.0.0.0/8 } - - ipBlock: { cidr: 172.16.0.0/12 } - - ipBlock: { cidr: 192.168.0.0/16 } - - ipBlock: { cidr: 169.254.0.0/16 } - {{- end }} + {{- include "gameplane.apiServerEgressPeers" . | trim | nindent 8 }} ports: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 503c95481..e3f48ac83 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -596,8 +596,12 @@ networkPolicies: # probe ports, e.g. [{ protocol: TCP, port: 25565 }]. probePorts: [] # CIDRs that hold the kube-apiserver endpoint(s) the agent sidecar - # patches its status heartbeat through. Same fallback semantics as - # kubeletCIDRs: unset means RFC1918 + link-local on TCP 443/6443. + # patches its status heartbeat through. Unset means RFC1918 + link-local + # plus the kubernetes Service endpoint addresses detected at install or + # upgrade time (e.g. a k3s node's public IP), on TCP 443/6443. Offline + # renders (helm template, Argo CD) cannot detect them: set the list + # explicitly there when the endpoint is public. Re-run helm upgrade if + # the API server endpoint changes. apiServerCIDRs: [] # Egress allowance for game pods to reach the public internet. Game # servers download their binaries, assets, and mods on first start diff --git a/docs/install.md b/docs/install.md index f0b5767d7..83d1fc053 100644 --- a/docs/install.md +++ b/docs/install.md @@ -240,7 +240,7 @@ Top-level knobs (see `values.yaml` for the full list): - `networkPolicies.gameIngress` — control ingress to game pods from external sources (players) - `gameIngress.enabled` — toggle player-ingress allowance (default `true`) - `gameIngress.fromCIDRs` — CIDRs from which player traffic is allowed (default `[0.0.0.0/0]`) - - `networkPolicies.apiServerCIDRs` — CIDRs allowed for agent heartbeat to kube-apiserver (defaults to all RFC1918 and link-local addresses on TCP 443/6443; customize to narrow to specific API endpoint(s)) + - `networkPolicies.apiServerCIDRs` — CIDRs allowed for agent heartbeat to kube-apiserver (defaults to all RFC1918 and link-local addresses plus the `kubernetes` Service endpoints detected at install/upgrade time, on TCP 443/6443; offline renders such as `helm template` or Argo CD cannot detect endpoints, so set this when the API endpoint is public; customize to narrow to specific API endpoint(s)) - `networkPolicies.gameEgress` — control public-internet egress for game pods (downloads, mod registries) - `gameEgress.enabled` — toggle public-egress allowance (default `true`) - `gameEgress.ports` — TCP ports for downloads (default 80, 443) diff --git a/docs/security.md b/docs/security.md index bf2a0a202..8795aca63 100644 --- a/docs/security.md +++ b/docs/security.md @@ -321,8 +321,9 @@ When `networkPolicies.enabled=true` (default) the chart applies: for status heartbeat (GameServer status patches). By default permits TCP 443 and 6443 to all addresses in RFC1918 and link-local ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 169.254.0.0/16), not limited to the apiserver - endpoint. Customize via `networkPolicies.apiServerCIDRs` to narrow to specific - API server addresses. + endpoint, plus the `kubernetes` Service ClusterIPs and endpoint addresses + the chart detects at install/upgrade time (as /32 or /128). Customize via + `networkPolicies.apiServerCIDRs` to narrow to specific API server addresses. - `allow-api-to-agent` — allows the API and operator pods (in the control-plane namespace) to reach every game pod's agent on TCP port 8090. The API proxies console/files/logs/players; the operator calls `/quiesce` From 63e5b20bb9446ce241b0a724e30c94c61a6902fd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 23:04:45 +0000 Subject: [PATCH 2/2] docs(changelog): move apiserver detection entry to changelog/unreleased Co-Authored-By: Claude Opus 5.5 Signed-off-by: Claude --- CHANGELOG.md | 8 -------- changelog/unreleased/930-detect-apiserver-endpoint.md | 7 +++++++ 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/930-detect-apiserver-endpoint.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 650c89421..a821a78ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,6 @@ See the [unreleased change entries](changelog/unreleased/) for individual fixes ## [Unreleased] -### Fixed - -- **Servers stuck in "Starting" when the API server has a public address:** - the chart's `allow-agent-to-apiserver` and `allow-sentinel-to-apiserver` - policies only allowed private ranges by default, so on a k3s node with a - public IP every agent heartbeat timed out. When - `networkPolicies.apiServerCIDRs` is unset, the chart now also allows the - `kubernetes` Service endpoints it detects at install or upgrade time. ### Added - The Files editor checks the open JavaScript or TypeScript file against the diff --git a/changelog/unreleased/930-detect-apiserver-endpoint.md b/changelog/unreleased/930-detect-apiserver-endpoint.md new file mode 100644 index 000000000..d4eba980c --- /dev/null +++ b/changelog/unreleased/930-detect-apiserver-endpoint.md @@ -0,0 +1,7 @@ +# Agent heartbeats reach a public API server endpoint + +The chart's `allow-agent-to-apiserver` and `allow-sentinel-to-apiserver` policies only allowed +private ranges by default, so on a k3s node with a public IP every agent heartbeat timed out +and servers stayed "Starting" with no CPU, memory or disk figures. When +`networkPolicies.apiServerCIDRs` is unset, the chart now also allows the `kubernetes` Service +addresses it detects at install or upgrade time. [PR #930](https://github.com/GameplanePanel/Gameplane/pull/930)