feat: add --watch-namespaces to scope the informer cache

cmd/main.go

@@ -22,6 +22,7 @@ import (
 	"flag"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -34,6 +35,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -73,6 +75,7 @@ func main() {
 	var discoveryBufferSize int
 	var kubeAPIQPS float64
 	var kubeAPIBurst int
+	var watchNamespaces string
 	flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.")
 	flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.")
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
@@ -84,6 +87,7 @@ func main() {
 	flag.IntVar(&discoveryBufferSize, "discovery-buffer-size", 10, "Amount of discovery messages that can be queued in the channel buffer.")
 	flag.Float64Var(&kubeAPIQPS, "kube-api-qps", 50, "Maximum sustained queries per second to the Kubernetes API server. The client-go default (20) is too low for large target populations.")
 	flag.IntVar(&kubeAPIBurst, "kube-api-burst", 100, "Maximum burst of queries to the Kubernetes API server.")
+	flag.StringVar(&watchNamespaces, "watch-namespaces", "", "Comma-separated list of namespaces to watch. Empty (the default) watches all namespaces, which caches every Secret, ConfigMap, Service, StatefulSet and Certificate in the cluster.")
 	opts := zap.Options{
 		Development: devMode,
 	}
@@ -102,8 +106,35 @@ func main() {
 	restConfig.Burst = kubeAPIBurst
 	setupLog.Info("configured Kubernetes API client rate limits", "qps", restConfig.QPS, "burst", restConfig.Burst)
 
+	// Restricting the cache to specific namespaces is the single largest reduction in
+	// informer footprint available: the manager caches every Secret, ConfigMap, Service,
+	// StatefulSet and Certificate it touches, cluster-wide, and Secrets in particular are
+	// unbounded and unrelated to Target count.
+	//
+	// This is safe to scope because every resolution path is already namespace-local — a
+	// Pipeline only ever resolves Targets, Subscriptions, Outputs, Inputs and Processors in
+	// its own namespace, and credentials are read from the Target's namespace. Nothing is
+	// read from the operator's own namespace: its TLS material comes from mounted files, not
+	// the API.
+	namespaces := parseWatchNamespaces(watchNamespaces)
+	cacheOpts := cache.Options{}
+	if len(namespaces) > 0 {
+		cacheOpts.DefaultNamespaces = make(map[string]cache.Config, len(namespaces))
+		for _, ns := range namespaces {
+			cacheOpts.DefaultNamespaces[ns] = cache.Config{}
+		}
+		setupLog.Info("restricting cache to namespaces", "namespaces", namespaces)
+	} else {
+		setupLog.Info("watching all namespaces; set --watch-namespaces to reduce cache footprint")
+	}
+	// The webhooks are registered cluster-wide, so they receive admission requests for
+	// namespaces this instance does not reconcile. Give them the same list so they can warn
+	// rather than silently accept resources that will never be acted on.
+	webhookv1alpha1.SetWatchedNamespaces(namespaces)
+
 	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
 		Scheme:                 scheme,
+		Cache:                  cacheOpts,
 		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
@@ -290,3 +321,22 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// parseWatchNamespaces splits the --watch-namespaces value into a deduplicated, ordered
+// list. An empty or whitespace-only value yields nil, meaning "watch all namespaces".
+func parseWatchNamespaces(value string) []string {
+	seen := make(map[string]struct{})
+	var namespaces []string
+	for _, ns := range strings.Split(value, ",") {
+		ns = strings.TrimSpace(ns)
+		if ns == "" {
+			continue
+		}
+		if _, ok := seen[ns]; ok {
+			continue
+		}
+		seen[ns] = struct{}{}
+		namespaces = append(namespaces, ns)
+	}
+	return namespaces
+}

cmd/main_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseWatchNamespaces(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		value string
+		want  []string
+	}{
+		// nil means "watch everything" — the flag being unset must not silently
+		// scope the cache to nothing.
+		{"empty", "", nil},
+		{"whitespace only", "   ", nil},
+		{"commas only", ",,", nil},
+		{"single", "alpha", []string{"alpha"}},
+		{"multiple", "alpha,beta", []string{"alpha", "beta"}},
+		{"trims spaces", " alpha , beta ", []string{"alpha", "beta"}},
+		{"drops empties", "alpha,,beta,", []string{"alpha", "beta"}},
+		{"dedupes, keeps order", "beta,alpha,beta", []string{"beta", "alpha"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := parseWatchNamespaces(tc.value); !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("parseWatchNamespaces(%q) = %v, want %v", tc.value, got, tc.want)
+			}
+		})
+	}
+}

docs/content/docs/advanced/operator-resources.md

@@ -0,0 +1,159 @@
+---
+title: "Operator Resources"
+linkTitle: "Operator Resources"
+weight: 3
+description: >
+  Sizing the operator pod and tuning its Kubernetes API usage at scale
+---
+
+## Overview
+
+[Scaling](../scaling/) covers sizing the gNMIc collector pods. This page is about the
+**operator pod itself** — how much memory it needs, why that is not driven by target count
+alone, and the two settings that matter most as a deployment grows.
+
+The short version: the operator's memory is dominated by what it *caches*, not by how many
+targets it manages. Restricting the cache with `--watch-namespaces` is the single largest
+reduction available.
+
+## Why the operator caches so much
+
+Like most controllers, the operator reads through a shared informer cache rather than
+hitting the API server on every lookup. An informer is started the first time any code path
+reads a type, and by default each one caches **every object of that type in the cluster**.
+
+The operator ends up caching six non-CRD types alongside its own CRDs:
+
+| Type | Why |
+|---|---|
+| `Secret` | Target credentials, cert-manager issuer CAs, TargetSource authentication |
+| `ConfigMap` | Per-cluster controller CA material |
+| `Service` | Prometheus output services, tunnel services |
+| `StatefulSet` | The gNMIc collector StatefulSet per Cluster |
+| `Certificate`, `Issuer` | cert-manager integration when TLS is enabled |
+
+`Secret` is usually the largest by a wide margin, and it has nothing to do with how many
+targets you have. Service-account tokens, TLS material and Helm release objects accumulate
+across a cluster, and all of them sit in the operator's memory whether or not it ever reads
+them.
+
+This is why the memory request cannot be derived from target count alone.
+
+## Restricting the watched namespaces
+
+**Flag**: `--watch-namespaces`
+**Helm Value**: `watchNamespaces`
+**Type**: Comma-separated string (Helm: list)
+**Default**: empty — watch all namespaces
+
+**What it does**: limits the informer cache to the listed namespaces. Objects outside them
+are neither cached nor watched.
+
+```yaml
+# values.yaml
+watchNamespaces:
+  - network-telemetry
+```
+
+```bash
+# equivalently
+--watch-namespaces=network-telemetry
+```
+
+Startup logs confirm the setting either way:
+
+```
+INFO  setup  restricting cache to namespaces  {"namespaces": ["network-telemetry"]}
+INFO  setup  watching all namespaces; set --watch-namespaces to reduce cache footprint
+```
+
+### Why this is safe
+
+The operator never resolves resources across namespace boundaries:
+
+- A `Pipeline` resolves `Target`, `Subscription`, `Output`, `Input`, `Processor` and
+  `TunnelTargetPolicy` objects only in its own namespace.
+- `credentialsRef` on a `TargetProfile` is resolved in the **Target's** namespace.
+- A `Cluster` reconciles only resources in its own namespace.
+
+There is no cross-namespace reference anywhere in the API, so each namespace is already
+self-contained. The operator also reads nothing from its *own* namespace — its TLS material
+comes from mounted files rather than the API — so you do not need to add the operator's
+namespace to the list.
+
+### What to watch out for
+
+**Resources in unwatched namespaces are silently ignored.** They are accepted by the API
+server and then never reconciled: no StatefulSet, no status, no events. To make this
+visible, creating a `Cluster`, `Pipeline` or `TargetSource` in an unwatched namespace
+produces an admission warning:
+
+```
+Warning: namespace "other-ns" is not watched by this gnmic-operator instance
+(watching: network-telemetry); this Cluster will be accepted but never reconciled
+```
+
+This is a warning rather than a rejection on purpose. Several namespace-scoped operator
+instances can share a cluster, and because the webhook configurations are registered
+cluster-wide, rejecting would mean every instance blocked resources intended for the others.
+
+**The ClusterRole is still cluster-scoped.** Restricting the cache does not narrow the
+operator's permissions. Converting to a `Role` per namespace is a separate change.
+
+**Each namespace adds an informer set.** Memory scales with the number of watched
+namespaces, so listing many namespaces recovers less than listing one.
+
+## Kubernetes API rate limits
+
+**Flags**: `--kube-api-qps`, `--kube-api-burst`
+**Helm Values**: `kubeApi.qps`, `kubeApi.burst`
+**Type**: Float, Integer
+**Defaults**: `50`, `100`
+
+**What it does**: sets the client-side rate limit for all Kubernetes API calls made by the
+operator process.
+
+The client-go defaults (20 QPS / 30 burst) are shared by every controller in the process.
+When one controller is busy — for example writing `Target` status for a large population —
+the others queue behind it. The symptom is misleading: `Cluster` reconciles appear
+inexplicably slow with no errors logged anywhere, because the delay is client-side
+throttling rather than API server pressure.
+
+Raise these if you see reconcile latency that does not correspond to API server load.
+Client-go logs waits longer than a second, and the `rest_client_rate_limiter_duration_seconds`
+metric shows time spent waiting on the limiter.
+
+## Memory sizing
+
+The chart ships:
+
+```yaml
+resources:
+  limits:
+    cpu: 500m
+    memory: 1Gi
+  requests:
+    cpu: 100m
+    memory: 256Mi
+```
+
+> These defaults are a starting point, not a measured figure. Because the footprint depends
+> on the total number of cached objects in your cluster rather than on target count, there
+> is no formula that holds across environments.
+
+Measure it in yours:
+
+```promql
+# Operator pod memory
+container_memory_working_set_bytes{pod=~"gnmic-operator-.*"}
+
+# Go heap, if the operator's metrics endpoint is scraped
+go_memstats_heap_inuse_bytes{job="gnmic-operator"}
+```
+
+Set the limit above the observed peak with headroom, and re-check after any significant
+growth in cluster-wide Secret count — not just after adding targets.
+
+If the operator is being OOM-killed, prefer `--watch-namespaces` over simply raising the
+limit. It addresses the cause rather than the symptom, and on a busy shared cluster the
+difference is usually large.

docs/content/docs/advanced/scaling.md

@@ -8,6 +8,10 @@ description: >
 
 The gNMIc Operator supports horizontal scaling of collector clusters. This page explains how scaling works and best practices for production deployments.
 
+> This page is about sizing the gNMIc **collector pods**. For the operator pod itself — its
+> memory footprint, `--watch-namespaces`, and Kubernetes API rate limits — see
+> [Operator Resources](../operator-resources/).
+
 ## Scaling a Cluster
 
 To scale a cluster, update the `replicas` field:

docs/content/docs/reference/helm-chart.md

@@ -122,20 +122,57 @@ affinity:
 | Parameter | Description | Default |
 |-----------|-------------|---------|
 | `resources.limits.cpu` | CPU limit | `500m` |
-| `resources.limits.memory` | Memory limit | `256Mi` |
-| `resources.requests.cpu` | CPU request | `10m` |
-| `resources.requests.memory` | Memory request | `64Mi` |
+| `resources.limits.memory` | Memory limit | `1Gi` |
+| `resources.requests.cpu` | CPU request | `100m` |
+| `resources.requests.memory` | Memory request | `256Mi` |
 
 ```yaml
 resources:
   limits:
     cpu: 1000m
-    memory: 512Mi
+    memory: 2Gi
   requests:
     cpu: 100m
-    memory: 128Mi
+    memory: 512Mi
+```
+
+Operator memory is driven by the number of objects it caches cluster-wide — Secrets in
+particular — not by how many Targets it manages. Treat these defaults as a starting point
+and measure. See [Operator Resources]({{< relref "../advanced/operator-resources" >}}).
+
+### Watched Namespaces
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `watchNamespaces` | Namespaces to watch. Empty watches the whole cluster | `[]` |
+
+```yaml
+watchNamespaces:
+  - network-telemetry
+```
+
+Restricting this is the largest single reduction in operator memory footprint. Resources in
+unwatched namespaces are accepted but never reconciled; creating a `Cluster`, `Pipeline` or
+`TargetSource` in one produces an admission warning. See
+[Operator Resources]({{< relref "../advanced/operator-resources" >}}).
+
+### Kubernetes API Rate Limits
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `kubeApi.qps` | Sustained queries per second to the Kubernetes API server | `50` |
+| `kubeApi.burst` | Maximum burst of queries to the Kubernetes API server | `100` |
+
+```yaml
+kubeApi:
+  qps: 100
+  burst: 200
 ```
 
+The client-go defaults (20 QPS / 30 burst) are shared by every controller in the process, so
+one busy controller throttles the rest. See
+[Operator Resources]({{< relref "../advanced/operator-resources" >}}).
+
 ### Leader Election
 
 | Parameter | Description | Default |

helm/templates/deployment.yaml

@@ -45,6 +45,9 @@ spec:
             - --discovery-buffer-size={{ .Values.discovery.bufferSize }}
             - --kube-api-qps={{ .Values.kubeApi.qps }}
             - --kube-api-burst={{ .Values.kubeApi.burst }}
+            {{- if .Values.watchNamespaces }}
+            - --watch-namespaces={{ join "," .Values.watchNamespaces }}
+            {{- end }}
           env:
             - name: POD_NAMESPACE
               valueFrom:

helm/values.yaml

@@ -54,6 +54,21 @@ kubeApi:
   qps: 50
   burst: 100
 
+# Namespaces to watch. Empty (the default) watches the whole cluster, which caches every
+# Secret, ConfigMap, Service, StatefulSet and Certificate in it — Secrets in particular are
+# unbounded and unrelated to how many Targets you have.
+#
+# Restricting this is the single largest reduction in memory footprint available. It is safe
+# because the operator never resolves across namespaces: a Pipeline only ever uses Targets,
+# Subscriptions, Outputs, Inputs and Processors from its own namespace, and credentials come
+# from the Target's namespace.
+#
+# The ClusterRole is left cluster-scoped either way; narrowing it to a Role per namespace is
+# a separate change.
+#   watchNamespaces:
+#     - network-telemetry
+watchNamespaces: []
+
 nodeSelector: {}
 
 tolerations: []

internal/webhook/v1alpha1/cluster_webhook.go

@@ -81,7 +81,7 @@ var _ admission.Validator[*operatorv1alpha1.Cluster] = &ClusterCustomValidator{}
 func (v *ClusterCustomValidator) ValidateCreate(_ context.Context, cluster *operatorv1alpha1.Cluster) (admission.Warnings, error) {
 	clusterlog.Info("Validation for Cluster upon creation", "name", cluster.GetName())
 
-	return nil, validateClusterSpec(&cluster.Spec)
+	return unwatchedNamespaceWarning("Cluster", cluster.GetNamespace()), validateClusterSpec(&cluster.Spec)
 }
 
 // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type Cluster.