Alert when a cell can't tell which location it serves

config/telemetry/alerts/cell-location.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: nso-cell-location
+  namespace: nso-slo
+spec:
+  groups:
+  - name: nso-cell-location
+    interval: 30s
+    rules:
+    - alert: CellLocationUnresolved
+      expr: max by (cluster, namespace, pod, reason) (nso_cell_location_identity_waiting) == 1
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: "{{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} cannot tell which location it serves"
+        description: "Pod '{{ $labels.pod }}' in cluster '{{ $labels.cluster }}', namespace '{{ $labels.namespace }}' has not been able to work out its location for 10 minutes ({{ $labels.reason }}). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations."
+        runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved"

config/telemetry/alerts/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - controller-health.yaml
   - trafficprotectionpolicies.yaml
   - server-config.yaml
+  - cell-location.yaml

docs/runbooks/cell-location.md

@@ -0,0 +1,33 @@
+# Runbook: Cell location alerts
+
+A cell controller manager works out the location it serves from the
+ServingLocation delivered to its cluster, or from `location.name` in its server
+config when none is delivered. It reports the result in
+`nso_cell_location_identity_waiting`, which is 1 while it cannot, labelled with
+the reason.
+
+## CellLocationUnresolved
+
+**Meaning (warning).** For 10 minutes, the cell could not name its location.
+The reason label says why:
+
+- `NoLocationIdentity`: no ServingLocation has been delivered and
+  `location.name` is not configured.
+- `AmbiguousLocationIdentity`: more than one ServingLocation has been delivered
+  and no location is configured to fall back to.
+
+**Impact.** The cell allocates no interface addresses and publishes no workload
+records to the federation hub. Workloads in this location keep running, but new
+interfaces stay unbound and HTTP proxies served from other locations cannot
+reach them.
+
+**Diagnose.**
+
+```sh
+kubectl get servinglocations
+kubectl -n <namespace> logs <pod> | grep 'knows its location'
+```
+
+**Remediate.** Deliver exactly one ServingLocation to the cluster, or set
+`location.name` in the cell's server config. The cell picks the location up as
+soon as it arrives, publishes what it was holding back, and the alert clears.

internal/controller/vpcendpointslice_writeback.go

@@ -468,9 +468,12 @@ func (r *VPCEndpointSliceWriteBackReconciler) republishNamespace(ctx context.Con
 // then.
 func (r *VPCEndpointSliceWriteBackReconciler) location(ctx context.Context) (string, error) {
 	identity, err := resolveLocationIdentity(ctx, r.localReader, r.Location)
+	reportLocationIdentity(identity, err)
 	if err != nil {
 		var unresolved *locationidentity.LocationUnresolved
 		if errors.As(err, &unresolved) {
+			log.FromContext(ctx).Error(err, "not publishing vpc endpointslices until this cell knows its location",
+				"reason", unresolved.Reason)
 			return "", nil
 		}
 		return "", err

test/prometheus-rules/cell-location/cell-location-rules.yaml

@@ -0,0 +1,13 @@
+groups:
+- name: nso-cell-location
+  interval: 30s
+  rules:
+  - alert: CellLocationUnresolved
+    expr: max by (cluster, namespace, pod, reason) (nso_cell_location_identity_waiting) == 1
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} cannot tell which location it serves"
+      description: "Pod '{{ $labels.pod }}' in cluster '{{ $labels.cluster }}', namespace '{{ $labels.namespace }}' has not been able to work out its location for 10 minutes ({{ $labels.reason }}). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations."
+      runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved"

test/prometheus-rules/cell-location/cell-location-tests.yaml

@@ -0,0 +1,36 @@
+rule_files:
+  - cell-location-rules.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: 'nso_cell_location_identity_waiting{cluster="lab",namespace="nso-system",pod="cell-controller-manager-0",reason="NoLocationIdentity"}'
+        values: '0 0 1x15'
+    alert_rule_test:
+      - eval_time: 11m
+        alertname: CellLocationUnresolved
+        exp_alerts: []
+      - eval_time: 13m
+        alertname: CellLocationUnresolved
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              cluster: lab
+              namespace: nso-system
+              pod: cell-controller-manager-0
+              reason: NoLocationIdentity
+            exp_annotations:
+              summary: "cell-controller-manager-0 in lab/nso-system cannot tell which location it serves"
+              description: "Pod 'cell-controller-manager-0' in cluster 'lab', namespace 'nso-system' has not been able to work out its location for 10 minutes (NoLocationIdentity). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations."
+              runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved"
+
+  - interval: 1m
+    input_series:
+      - series: 'nso_cell_location_identity_waiting{cluster="lab",namespace="nso-system",pod="cell-controller-manager-0",reason="NoLocationIdentity"}'
+        values: '1x5 0x10'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: CellLocationUnresolved
+        exp_alerts: []