feat(observability): qualify telemetry across managed tenants

CHANGELOG.md

@@ -5,6 +5,8 @@ All notable changes to Gameplane are documented here. The format is based on
 to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) once it
 reaches `1.0.0`. Pre-1.0 minor versions may contain breaking changes.
 
+See the [unreleased change entries](changelog/unreleased/) for individual fixes and upgrade notes.
+
 ## [Unreleased]
 
 ## [0.4.0] — 2026-11-10

changelog/unreleased/908-tenant-monitoring.md

@@ -0,0 +1,4 @@
+# Monitoring across managed tenants
+
+Prometheus discovers game workloads in managed namespaces and distinguishes missing targets
+from failed scrapes. Dashboards show process RSS and container working set separately. [PR #908](https://github.com/GameplanePanel/Gameplane/pull/908)

charts/gameplane/dashboards/gameplane-workloads.json

@@ -340,6 +340,46 @@
           "legendFormat": "{{namespace}} / {{pod}} {{metric}}"
         }
       ]
+    },
+    {
+      "id": 11,
+      "title": "Relay drops and rejected flows",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": { "defaults": { "unit": "ops", "noValue": "Unknown", "custom": { "spanNulls": false } } },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (namespace,pod,protocol) (rate(gameplane_tunnel_flow_rejections_total{namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": "{{namespace}} / {{pod}} {{protocol}} rejections"
+        },
+        {
+          "refId": "B",
+          "expr": "max by (namespace,pod) (rate(gameplane_tunnel_udp_queue_drops_total{namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": "{{namespace}} / {{pod}} UDP queue drops"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "title": "Game process RSS and container working set",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "noValue": "Unknown", "custom": { "spanNulls": false } } },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (namespace,pod) (gameplane_agent_memory_bytes{namespace=~\"$namespace\"})",
+          "legendFormat": "{{namespace}} / {{pod}} process RSS"
+        },
+        {
+          "refId": "B",
+          "expr": "max by (namespace,pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"game\"})",
+          "legendFormat": "{{namespace}} / {{pod}} working set"
+        }
+      ]
     }
   ]
 }

charts/gameplane/templates/monitoring-rules.yaml

@@ -1,4 +1,5 @@
 {{- if and .Values.prometheusRules.enabled (eq (include "gameplane.operatorEnabled" .) "true") }}
+{{- $namespaces := join "|" (include "gameplane.operatorGameNamespaces" . | fromJsonArray) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -13,59 +14,79 @@ spec:
   groups:
     - name: gameplane.workloads
       rules:
+        - alert: GameplaneScrapeTargetMissing
+          expr: |
+            (
+              max by (namespace, pod, gameplane_component) (
+                label_replace(kube_pod_container_status_running{namespace=~"{{ $namespaces }}",container=~"agent|tunnel"} == 1,
+                  "gameplane_component", "$1", "container", "(agent|tunnel)")
+              )
+              and on (namespace, pod) kube_pod_labels{namespace=~"{{ $namespaces }}",label_app_kubernetes_io_name=~"gameplane-game|gameplane-tunnel"}
+            )
+            unless on (namespace, pod, gameplane_component) up{namespace=~"{{ $namespaces }}",gameplane_component=~"agent|tunnel"}
+          for: 5m
+          labels: { severity: warning }
+          annotations: { summary: "Running Gameplane workload is missing from scrape discovery" }
         - alert: GameplaneScrapeUnavailable
-          expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace="{{ .Values.gamesNamespace }}"}) == 0
+          expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace=~"{{ $namespaces }}"}) == 0
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "Gameplane metrics target cannot be scraped" }
         - alert: GameplaneRelayUnavailable
-          expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 1)
+          expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace=~"{{ $namespaces }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace=~"{{ $namespaces }}"}) == 1)
           for: 2m
           labels: { severity: warning }
           annotations: { summary: "Gameplane relay connection or forwards are unavailable" }
         - alert: GameplaneRelayHealthUnknown
-          expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace="{{ .Values.gamesNamespace }}"}) == 1)
+          expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace=~"{{ $namespaces }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace=~"{{ $namespaces }}"}) == 1)
           for: 5m
           labels: { severity: info }
           annotations: { summary: "Gameplane provider cannot confirm live relay health" }
+        - alert: GameplaneRelaySaturated
+          expr: |
+            max by (namespace, pod) (increase(gameplane_tunnel_flow_rejections_total{namespace=~"{{ $namespaces }}"}[5m])) > 0
+            or
+            max by (namespace, pod) (increase(gameplane_tunnel_udp_queue_drops_total{namespace=~"{{ $namespaces }}"}[5m])) > 0
+          labels: { severity: warning }
+          annotations: { summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets" }
         - alert: GameplanePerformanceUnknown
-          expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace="{{ .Values.gamesNamespace }}"}) == 0
+          expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace=~"{{ $namespaces }}"}) == 0
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "Declared game performance measurement is unavailable" }
         - alert: GameplanePerformanceDegraded
           expr: |
-            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"})
+            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace=~"{{ $namespaces }}"})
               < on (namespace, pod, metric, signal, unit)
-             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="below"}))
+             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace=~"{{ $namespaces }}",direction="below"}))
             or
-            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"})
+            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace=~"{{ $namespaces }}"})
               > on (namespace, pod, metric, signal, unit)
-             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="above"}))
+             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace=~"{{ $namespaces }}",direction="above"}))
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "Game performance crossed its module-declared threshold" }
         - alert: GameplaneCPUThrottled
-          expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m])) > 0.25
+          expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace=~"{{ $namespaces }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace=~"{{ $namespaces }}",container="game"}[5m])) > 0.25
           for: 10m
           labels: { severity: warning }
           annotations: { summary: "Game container CPU is throttled in more than 25% of periods" }
         - alert: GameplaneMemoryPressure
-          expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} / (container_spec_memory_limit_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} > 0)) > 0.9
+          expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"{{ $namespaces }}",container="game"} / (container_spec_memory_limit_bytes{namespace=~"{{ $namespaces }}",container="game"} > 0)) > 0.9
           for: 10m
           labels: { severity: warning }
           annotations: { summary: "Game container working set exceeds 90% of its memory limit" }
         - alert: GameplaneContainerOOM
-          expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="{{ .Values.gamesNamespace }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace="{{ .Values.gamesNamespace }}",reason="OOMKilled"}) == 1)
+          expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace=~"{{ $namespaces }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $namespaces }}",reason="OOMKilled"}) == 1)
           labels: { severity: warning }
           annotations: { summary: "Gameplane container restarted after an OOM kill" }
         - alert: GameplanePodPending
-          expr: max by (namespace, pod) (kube_pod_status_phase{namespace="{{ .Values.gamesNamespace }}",phase="Pending"}) == 1
+          expr: max by (namespace, pod) (kube_pod_status_phase{namespace=~"{{ $namespaces }}",phase="Pending"}) == 1
           for: 10m
           labels: { severity: warning }
           annotations: { summary: "Gameplane Pod has remained Pending for ten minutes" }
         - alert: GameplaneNodePressure
-          expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace="{{ .Values.gamesNamespace }}"})
+          expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace=~"{{ $namespaces }}"})
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "A node hosting Gameplane Pods reports resource pressure" }

charts/gameplane/templates/servicemonitors.yaml

@@ -101,6 +101,8 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
 spec:
+  namespaceSelector:
+    matchNames: {{ include "gameplane.operatorGameNamespaces" . }}
   selector:
     matchLabels: { app.kubernetes.io/name: gameplane-game }
   podMetricsEndpoints:
@@ -124,6 +126,8 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
 spec:
+  namespaceSelector:
+    matchNames: {{ include "gameplane.operatorGameNamespaces" . }}
   selector:
     matchLabels: { app.kubernetes.io/name: gameplane-tunnel }
   podMetricsEndpoints:

charts/gameplane/tests/monitoring-rules.sh

@@ -4,14 +4,20 @@ set -euo pipefail
 cd "$(dirname "$0")/../../.."
 scratch="$(mktemp -d)"
 trap 'rm -rf "$scratch"' EXIT
-helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml > "$scratch/rendered.yaml"
+helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml \
+  --set 'operator.gameNamespaces={tenant-b,gameplane-games}' > "$scratch/rendered.yaml"
 python3 - "$scratch" <<'PY'
 import pathlib,sys,yaml
 root=pathlib.Path(sys.argv[1])
 groups=[]
+monitors={}
 for document in yaml.safe_load_all((root/'rendered.yaml').read_text()):
     if document and document.get('kind')=='PrometheusRule':
         groups.extend(document['spec']['groups'])
+    if document and document.get('kind')=='PodMonitor':
+        monitors[document['metadata']['name']]=document['spec']['namespaceSelector']['matchNames']
+expected=['gameplane-games','tenant-b']
+assert monitors=={'gameplane-agent':expected,'gameplane-tunnel':expected}, monitors
 (root/'rules.yaml').write_text(yaml.safe_dump({'groups':groups}))
 PY
 cp charts/gameplane/tests/monitoring-rules.test.yaml "$scratch/tests.yaml"

charts/gameplane/tests/monitoring-rules.test.yaml

@@ -1,6 +1,54 @@
 rule_files: [rules.yaml]
 evaluation_interval: 1m
 tests:
+  - interval: 1m
+    input_series:
+      - series: 'gameplane_tunnel_flow_rejections_total{namespace="gameplane-games",pod="busy",protocol="tcp"}'
+        values: '0x2 1x10'
+      - series: 'gameplane_tunnel_udp_queue_drops_total{namespace="tenant-b",pod="backlogged"}'
+        values: '0x2 1x10'
+      - series: 'gameplane_tunnel_udp_queue_drops_total{namespace="unmanaged",pod="outside"}'
+        values: '0+10x12'
+      - series: 'gameplane_tunnel_forward_metrics_supported{namespace="tenant-b",pod="opaque"}'
+        values: '0x12'
+    alert_rule_test:
+      - eval_time: 4m
+        alertname: GameplaneRelaySaturated
+        exp_alerts:
+          - exp_labels: {namespace: gameplane-games, pod: busy, severity: warning}
+            exp_annotations: {summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets"}
+          - exp_labels: {namespace: tenant-b, pod: backlogged, severity: warning}
+            exp_annotations: {summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets"}
+      - eval_time: 10m
+        alertname: GameplaneRelaySaturated
+        exp_alerts: []
+  - interval: 1m
+    input_series:
+      - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="sleepy-0",container="agent"}'
+        values: '1x12'
+      - series: 'kube_pod_labels{namespace="tenant-b",pod="sleepy-0",label_app_kubernetes_io_name="gameplane-game"}'
+        values: '1x12'
+      - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="relay",container="tunnel"}'
+        values: '1x12'
+      - series: 'kube_pod_labels{namespace="tenant-b",pod="relay",label_app_kubernetes_io_name="gameplane-tunnel"}'
+        values: '1x12'
+      - series: 'up{namespace="tenant-b",pod="relay",gameplane_component="tunnel"}'
+        values: '0x12'
+      - series: 'kube_pod_container_status_running{namespace="unmanaged",pod="hidden",container="agent"}'
+        values: '1x12'
+      - series: 'kube_pod_labels{namespace="unmanaged",pod="hidden",label_app_kubernetes_io_name="gameplane-game"}'
+        values: '1x12'
+    alert_rule_test:
+      - eval_time: 8m
+        alertname: GameplaneScrapeTargetMissing
+        exp_alerts:
+          - exp_labels: {namespace: tenant-b, pod: sleepy-0, gameplane_component: agent, severity: warning}
+            exp_annotations: {summary: "Running Gameplane workload is missing from scrape discovery"}
+      - eval_time: 8m
+        alertname: GameplaneScrapeUnavailable
+        exp_alerts:
+          - exp_labels: {namespace: tenant-b, pod: relay, gameplane_component: tunnel, severity: warning}
+            exp_annotations: {summary: "Gameplane metrics target cannot be scraped"}
   - interval: 1m
     input_series:
       - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds"}'