diff --git a/config/telemetry/alerts/cell-location.yaml b/config/telemetry/alerts/cell-location.yaml new file mode 100644 index 00000000..b71c7556 --- /dev/null +++ b/config/telemetry/alerts/cell-location.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: nso-cell-location + namespace: nso-slo +spec: + groups: + - name: nso-cell-location + interval: 30s + rules: + - alert: CellLocationUnresolved + expr: max by (cluster, namespace, pod, reason) (nso_cell_location_identity_waiting) == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} cannot tell which location it serves" + description: "Pod '{{ $labels.pod }}' in cluster '{{ $labels.cluster }}', namespace '{{ $labels.namespace }}' has not been able to work out its location for 10 minutes ({{ $labels.reason }}). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved" diff --git a/config/telemetry/alerts/kustomization.yaml b/config/telemetry/alerts/kustomization.yaml index 433f7053..316adfb5 100644 --- a/config/telemetry/alerts/kustomization.yaml +++ b/config/telemetry/alerts/kustomization.yaml @@ -6,3 +6,4 @@ resources: - controller-health.yaml - trafficprotectionpolicies.yaml - server-config.yaml + - cell-location.yaml diff --git a/docs/runbooks/cell-location.md b/docs/runbooks/cell-location.md new file mode 100644 index 00000000..fd94e6ce --- /dev/null +++ b/docs/runbooks/cell-location.md @@ -0,0 +1,33 @@ +# Runbook: Cell location alerts + +A cell controller manager works out the location it serves from the +ServingLocation delivered to its cluster, or from `location.name` in its server +config when none is delivered. It reports the result in +`nso_cell_location_identity_waiting`, which is 1 while it cannot, labelled with +the reason. + +## CellLocationUnresolved + +**Meaning (warning).** For 10 minutes, the cell could not name its location. +The reason label says why: + +- `NoLocationIdentity`: no ServingLocation has been delivered and + `location.name` is not configured. +- `AmbiguousLocationIdentity`: more than one ServingLocation has been delivered + and no location is configured to fall back to. + +**Impact.** The cell allocates no interface addresses and publishes no workload +records to the federation hub. Workloads in this location keep running, but new +interfaces stay unbound and HTTP proxies served from other locations cannot +reach them. + +**Diagnose.** + +```sh +kubectl get servinglocations +kubectl -n logs | grep 'knows its location' +``` + +**Remediate.** Deliver exactly one ServingLocation to the cluster, or set +`location.name` in the cell's server config. The cell picks the location up as +soon as it arrives, publishes what it was holding back, and the alert clears. diff --git a/internal/controller/vpcendpointslice_writeback.go b/internal/controller/vpcendpointslice_writeback.go index 6c838948..4c1262bb 100644 --- a/internal/controller/vpcendpointslice_writeback.go +++ b/internal/controller/vpcendpointslice_writeback.go @@ -468,9 +468,12 @@ func (r *VPCEndpointSliceWriteBackReconciler) republishNamespace(ctx context.Con // then. func (r *VPCEndpointSliceWriteBackReconciler) location(ctx context.Context) (string, error) { identity, err := resolveLocationIdentity(ctx, r.localReader, r.Location) + reportLocationIdentity(identity, err) if err != nil { var unresolved *locationidentity.LocationUnresolved if errors.As(err, &unresolved) { + log.FromContext(ctx).Error(err, "not publishing vpc endpointslices until this cell knows its location", + "reason", unresolved.Reason) return "", nil } return "", err diff --git a/test/prometheus-rules/cell-location/cell-location-rules.yaml b/test/prometheus-rules/cell-location/cell-location-rules.yaml new file mode 100644 index 00000000..dd934b54 --- /dev/null +++ b/test/prometheus-rules/cell-location/cell-location-rules.yaml @@ -0,0 +1,13 @@ +groups: +- name: nso-cell-location + interval: 30s + rules: + - alert: CellLocationUnresolved + expr: max by (cluster, namespace, pod, reason) (nso_cell_location_identity_waiting) == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} cannot tell which location it serves" + description: "Pod '{{ $labels.pod }}' in cluster '{{ $labels.cluster }}', namespace '{{ $labels.namespace }}' has not been able to work out its location for 10 minutes ({{ $labels.reason }}). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved" diff --git a/test/prometheus-rules/cell-location/cell-location-tests.yaml b/test/prometheus-rules/cell-location/cell-location-tests.yaml new file mode 100644 index 00000000..fdb2e338 --- /dev/null +++ b/test/prometheus-rules/cell-location/cell-location-tests.yaml @@ -0,0 +1,36 @@ +rule_files: + - cell-location-rules.yaml + +evaluation_interval: 1m + +tests: + - interval: 1m + input_series: + - series: 'nso_cell_location_identity_waiting{cluster="lab",namespace="nso-system",pod="cell-controller-manager-0",reason="NoLocationIdentity"}' + values: '0 0 1x15' + alert_rule_test: + - eval_time: 11m + alertname: CellLocationUnresolved + exp_alerts: [] + - eval_time: 13m + alertname: CellLocationUnresolved + exp_alerts: + - exp_labels: + severity: warning + cluster: lab + namespace: nso-system + pod: cell-controller-manager-0 + reason: NoLocationIdentity + exp_annotations: + summary: "cell-controller-manager-0 in lab/nso-system cannot tell which location it serves" + description: "Pod 'cell-controller-manager-0' in cluster 'lab', namespace 'nso-system' has not been able to work out its location for 10 minutes (NoLocationIdentity). Until it can, it allocates no interface addresses and publishes no workload records to the edge, so HTTP proxies to workloads in this location cannot reach them from other locations." + runbook_url: "https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/cell-location.md#celllocationunresolved" + + - interval: 1m + input_series: + - series: 'nso_cell_location_identity_waiting{cluster="lab",namespace="nso-system",pod="cell-controller-manager-0",reason="NoLocationIdentity"}' + values: '1x5 0x10' + alert_rule_test: + - eval_time: 14m + alertname: CellLocationUnresolved + exp_alerts: []