From 0ef4da8a3a9148d3a6746587b504d4a0e2ae8d01 Mon Sep 17 00:00:00 2001 From: Scot Schuchert-Wells Date: Wed, 11 Nov 2026 14:53:24 +0000 Subject: [PATCH] feat: alert when a cell cannot name its location A cell that cannot work out its location allocates no interface addresses and publishes no workload records, but the write-back said nothing about it. It now logs an error and reports the waiting metric, and a new alert fires when the metric stays set for 10 minutes. Key changes: - vpc write-back reports location identity and logs while unresolved - CellLocationUnresolved alert with promtool tests and a runbook --- config/telemetry/alerts/cell-location.yaml | 19 ++++++++++ config/telemetry/alerts/kustomization.yaml | 1 + docs/runbooks/cell-location.md | 33 +++++++++++++++++ .../controller/vpcendpointslice_writeback.go | 3 ++ .../cell-location/cell-location-rules.yaml | 13 +++++++ .../cell-location/cell-location-tests.yaml | 36 +++++++++++++++++++ 6 files changed, 105 insertions(+) create mode 100644 config/telemetry/alerts/cell-location.yaml create mode 100644 docs/runbooks/cell-location.md create mode 100644 test/prometheus-rules/cell-location/cell-location-rules.yaml create mode 100644 test/prometheus-rules/cell-location/cell-location-tests.yaml diff --git a/config/telemetry/alerts/cell-location.yaml b/config/telemetry/alerts/cell-location.yaml new file mode 100644 index 00000000..b71c7556 --- /dev/null +++ b/config/telemetry/alerts/cell-location.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: nso-cell-location + namespace: nso-slo +spec: + groups: + - name: nso-cell-location + interval: 30s + rules: + - alert: CellLocationUnresolved + expr: max by (cluster, namespace, pod, reason) (nso_cell_location_identity_waiting) == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} cannot tell which location it serves" + description: "Pod '{{ $labels.pod }}' in cluster '{{ $labels.cluster }}', namespace '{{ $labels.namespace }}' has not been able to work out its location for 10 minutes ({{ $labels.reason }}). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved" diff --git a/config/telemetry/alerts/kustomization.yaml b/config/telemetry/alerts/kustomization.yaml index 433f7053..316adfb5 100644 --- a/config/telemetry/alerts/kustomization.yaml +++ b/config/telemetry/alerts/kustomization.yaml @@ -6,3 +6,4 @@ resources: - controller-health.yaml - trafficprotectionpolicies.yaml - server-config.yaml + - cell-location.yaml diff --git a/docs/runbooks/cell-location.md b/docs/runbooks/cell-location.md new file mode 100644 index 00000000..fd94e6ce --- /dev/null +++ b/docs/runbooks/cell-location.md @@ -0,0 +1,33 @@ +# Runbook: Cell location alerts + +A cell controller manager works out the location it serves from the +ServingLocation delivered to its cluster, or from `location.name` in its server +config when none is delivered. It reports the result in +`nso_cell_location_identity_waiting`, which is 1 while it cannot, labelled with +the reason. + +## CellLocationUnresolved + +**Meaning (warning).** For 10 minutes, the cell could not name its location. +The reason label says why: + +- `NoLocationIdentity`: no ServingLocation has been delivered and + `location.name` is not configured. +- `AmbiguousLocationIdentity`: more than one ServingLocation has been delivered + and no location is configured to fall back to. + +**Impact.** The cell allocates no interface addresses and publishes no workload +records to the federation hub. Workloads in this location keep running, but new +interfaces stay unbound and HTTP proxies served from other locations cannot +reach them. + +**Diagnose.** + +```sh +kubectl get servinglocations +kubectl -n logs | grep 'knows its location' +``` + +**Remediate.** Deliver exactly one ServingLocation to the cluster, or set +`location.name` in the cell's server config. The cell picks the location up as +soon as it arrives, publishes what it was holding back, and the alert clears. diff --git a/internal/controller/vpcendpointslice_writeback.go b/internal/controller/vpcendpointslice_writeback.go index 6c838948..4c1262bb 100644 --- a/internal/controller/vpcendpointslice_writeback.go +++ b/internal/controller/vpcendpointslice_writeback.go @@ -468,9 +468,12 @@ func (r *VPCEndpointSliceWriteBackReconciler) republishNamespace(ctx context.Con // then. func (r *VPCEndpointSliceWriteBackReconciler) location(ctx context.Context) (string, error) { identity, err := resolveLocationIdentity(ctx, r.localReader, r.Location) + reportLocationIdentity(identity, err) if err != nil { var unresolved *locationidentity.LocationUnresolved if errors.As(err, &unresolved) { + log.FromContext(ctx).Error(err, "not publishing vpc endpointslices until this cell knows its location", + "reason", unresolved.Reason) return "", nil } return "", err diff --git a/test/prometheus-rules/cell-location/cell-location-rules.yaml b/test/prometheus-rules/cell-location/cell-location-rules.yaml new file mode 100644 index 00000000..dd934b54 --- /dev/null +++ b/test/prometheus-rules/cell-location/cell-location-rules.yaml @@ -0,0 +1,13 @@ +groups: +- name: nso-cell-location + interval: 30s + rules: + - alert: CellLocationUnresolved + expr: max by (cluster, namespace, pod, reason) (nso_cell_location_identity_waiting) == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} cannot tell which location it serves" + description: "Pod '{{ $labels.pod }}' in cluster '{{ $labels.cluster }}', namespace '{{ $labels.namespace }}' has not been able to work out its location for 10 minutes ({{ $labels.reason }}). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved" diff --git a/test/prometheus-rules/cell-location/cell-location-tests.yaml b/test/prometheus-rules/cell-location/cell-location-tests.yaml new file mode 100644 index 00000000..fdb2e338 --- /dev/null +++ b/test/prometheus-rules/cell-location/cell-location-tests.yaml @@ -0,0 +1,36 @@ +rule_files: + - cell-location-rules.yaml + +evaluation_interval: 1m + +tests: + - interval: 1m + input_series: + - series: 'nso_cell_location_identity_waiting{cluster="lab",namespace="nso-system",pod="cell-controller-manager-0",reason="NoLocationIdentity"}' + values: '0 0 1x15' + alert_rule_test: + - eval_time: 11m + alertname: CellLocationUnresolved + exp_alerts: [] + - eval_time: 13m + alertname: CellLocationUnresolved + exp_alerts: + - exp_labels: + severity: warning + cluster: lab + namespace: nso-system + pod: cell-controller-manager-0 + reason: NoLocationIdentity + exp_annotations: + summary: "cell-controller-manager-0 in lab/nso-system cannot tell which location it serves" + description: "Pod 'cell-controller-manager-0' in cluster 'lab', namespace 'nso-system' has not been able to work out its location for 10 minutes (NoLocationIdentity). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved" + + - interval: 1m + input_series: + - series: 'nso_cell_location_identity_waiting{cluster="lab",namespace="nso-system",pod="cell-controller-manager-0",reason="NoLocationIdentity"}' + values: '1x5 0x10' + alert_rule_test: + - eval_time: 14m + alertname: CellLocationUnresolved + exp_alerts: []