diff --git a/changelog/unreleased/930-detect-apiserver-endpoint.md b/changelog/unreleased/930-detect-apiserver-endpoint.md new file mode 100644 index 000000000..d4eba980c --- /dev/null +++ b/changelog/unreleased/930-detect-apiserver-endpoint.md @@ -0,0 +1,7 @@ +# Agent heartbeats reach a public API server endpoint + +The chart's `allow-agent-to-apiserver` and `allow-sentinel-to-apiserver` policies only allowed +private ranges by default, so on a k3s node with a public IP every agent heartbeat timed out +and servers stayed "Starting" with no CPU, memory or disk figures. When +`networkPolicies.apiServerCIDRs` is unset, the chart now also allows the `kubernetes` Service +addresses it detects at install or upgrade time. [PR #930](https://github.com/GameplanePanel/Gameplane/pull/930) diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index 44bf31ee3..bb80d1b29 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -189,3 +189,47 @@ mergeOverwrite preserves explicit zero/empty values so validation can reject the {{- end -}} {{- uniq $namespaces | toJson -}} {{- end -}} + +{{- /* +gameplane.apiServerEgressPeers renders the NetworkPolicy `to:` peers of the +agent and sentinel apiserver egress rules. An explicit +networkPolicies.apiServerCIDRs list wins. Otherwise it renders the RFC1918 + +link-local fallback plus every address the live cluster reports for the +`kubernetes` Service: its ClusterIPs and its EndpointSlice endpoint addresses. +On k3s and kubeadm single-node installs that endpoint is the node's own IP, +which is often public and outside the fallback; without it heartbeats time +out and every server stays "Starting". Offline renders (helm template, client +dry-run) get empty lookups and keep the plain fallback. Addresses are read at +install/upgrade time, so re-run helm upgrade if the API endpoint moves. +*/}} +{{- define "gameplane.apiServerEgressPeers" -}} +{{- $cidrs := .Values.networkPolicies.apiServerCIDRs | default (list) -}} +{{- if not $cidrs -}} +{{- $cidrs = list "10.0.0.0/8" "172.16.0.0/12" "192.168.0.0/16" "169.254.0.0/16" -}} +{{- $addrs := list -}} +{{- $svc := lookup "v1" "Service" "default" "kubernetes" -}} +{{- range (dig "spec" "clusterIPs" (list) $svc) -}} +{{- $addrs = append $addrs . -}} +{{- end -}} +{{- $slices := lookup "discovery.k8s.io/v1" "EndpointSlice" "default" "" -}} +{{- range (dig "items" (list) $slices) -}} +{{- if eq (dig "metadata" "labels" "kubernetes.io/service-name" "" .) "kubernetes" -}} +{{- range (dig "endpoints" (list) .) -}} +{{- range (dig "addresses" (list) .) -}} +{{- $addrs = append $addrs . -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range $addrs -}} +{{- if contains ":" . -}} +{{- $cidrs = append $cidrs (printf "%s/128" .) -}} +{{- else -}} +{{- $cidrs = append $cidrs (printf "%s/32" .) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range (uniq $cidrs) }} +- ipBlock: { cidr: {{ . | quote }} } +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index ed2ae0a8a..3d8942297 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -40,9 +40,10 @@ spec: # Allow the agent sidecar to reach the kube-apiserver for its status # heartbeat (gameservers/status patches). Service VIP traffic is DNATed # to the apiserver endpoint before policy evaluation on most CNIs, so an -# ipBlock toward the endpoint address is required; like the kubelet -# probes policy below, default to RFC1918 + link-local and let operators -# narrow it via networkPolicies.apiServerCIDRs. +# ipBlock toward the endpoint address is required. Unset +# networkPolicies.apiServerCIDRs means RFC1918 + link-local plus the +# kubernetes Service endpoints detected at install time (a k3s node's +# public IP, for example); set the list to narrow it. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -56,17 +57,7 @@ spec: policyTypes: [Egress] egress: - to: - {{- $apiCidrs := (.Values.networkPolicies.apiServerCIDRs | default (list)) }} - {{- if $apiCidrs }} - {{- range $apiCidrs }} - - ipBlock: { cidr: {{ . | quote }} } - {{- end }} - {{- else }} - - ipBlock: { cidr: 10.0.0.0/8 } - - ipBlock: { cidr: 172.16.0.0/12 } - - ipBlock: { cidr: 192.168.0.0/16 } - - ipBlock: { cidr: 169.254.0.0/16 } - {{- end }} + {{- include "gameplane.apiServerEgressPeers" . | trim | nindent 8 }} ports: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } @@ -341,17 +332,7 @@ spec: policyTypes: [Egress] egress: - to: - {{- $apiCidrs := (.Values.networkPolicies.apiServerCIDRs | default (list)) }} - {{- if $apiCidrs }} - {{- range $apiCidrs }} - - ipBlock: { cidr: {{ . | quote }} } - {{- end }} - {{- else }} - - ipBlock: { cidr: 10.0.0.0/8 } - - ipBlock: { cidr: 172.16.0.0/12 } - - ipBlock: { cidr: 192.168.0.0/16 } - - ipBlock: { cidr: 169.254.0.0/16 } - {{- end }} + {{- include "gameplane.apiServerEgressPeers" . | trim | nindent 8 }} ports: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index f27559e43..3c9b909bb 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -599,8 +599,12 @@ networkPolicies: # probe ports, e.g. [{ protocol: TCP, port: 25565 }]. probePorts: [] # CIDRs that hold the kube-apiserver endpoint(s) the agent sidecar - # patches its status heartbeat through. Same fallback semantics as - # kubeletCIDRs: unset means RFC1918 + link-local on TCP 443/6443. + # patches its status heartbeat through. Unset means RFC1918 + link-local + # plus the kubernetes Service endpoint addresses detected at install or + # upgrade time (e.g. a k3s node's public IP), on TCP 443/6443. Offline + # renders (helm template, Argo CD) cannot detect them: set the list + # explicitly there when the endpoint is public. Re-run helm upgrade if + # the API server endpoint changes. apiServerCIDRs: [] # Egress allowance for game pods to reach the public internet. Game # servers download their binaries, assets, and mods on first start diff --git a/docs/install.md b/docs/install.md index 803bf5085..87b49085d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -240,7 +240,7 @@ Top-level knobs (see `values.yaml` for the full list): - `networkPolicies.gameIngress` — control ingress to game pods from external sources (players) - `gameIngress.enabled` — toggle player-ingress allowance (default `true`) - `gameIngress.fromCIDRs` — CIDRs from which player traffic is allowed (default `[0.0.0.0/0]`) - - `networkPolicies.apiServerCIDRs` — CIDRs allowed for agent heartbeat to kube-apiserver (defaults to all RFC1918 and link-local addresses on TCP 443/6443; customize to narrow to specific API endpoint(s)) + - `networkPolicies.apiServerCIDRs` — CIDRs allowed for agent heartbeat to kube-apiserver (defaults to all RFC1918 and link-local addresses plus the `kubernetes` Service endpoints detected at install/upgrade time, on TCP 443/6443; offline renders such as `helm template` or Argo CD cannot detect endpoints, so set this when the API endpoint is public; customize to narrow to specific API endpoint(s)) - `networkPolicies.gameEgress` — control public-internet egress for game pods (downloads, mod registries) - `gameEgress.enabled` — toggle public-egress allowance (default `true`) - `gameEgress.ports` — TCP ports for downloads (default 80, 443) diff --git a/docs/security.md b/docs/security.md index a67d58424..57b4f1d34 100644 --- a/docs/security.md +++ b/docs/security.md @@ -321,8 +321,9 @@ When `networkPolicies.enabled=true` (default) the chart applies: for status heartbeat (GameServer status patches). By default permits TCP 443 and 6443 to all addresses in RFC1918 and link-local ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 169.254.0.0/16), not limited to the apiserver - endpoint. Customize via `networkPolicies.apiServerCIDRs` to narrow to specific - API server addresses. + endpoint, plus the `kubernetes` Service ClusterIPs and endpoint addresses + the chart detects at install/upgrade time (as /32 or /128). Customize via + `networkPolicies.apiServerCIDRs` to narrow to specific API server addresses. - `allow-api-to-agent` — allows the API and operator pods (in the control-plane namespace) to reach every game pod's agent on TCP port 8090. The API proxies console/files/logs/players; the operator calls `/quiesce`