fix: Take a controller's owner from its namespace

config/telemetry/alerts/controller-health.yaml

@@ -28,7 +28,9 @@ spec:
       expr: |
         max without (datum_service) (label_replace(controller_runtime:reconcile_stalled:warning * on (cluster, namespace, deployment) group_left (datum_service, team) deployment:datum_service:info, "service", "$1", "datum_service", "(.*)"))
         or
-        (controller_runtime:reconcile_stalled:warning unless on (cluster, namespace, deployment) deployment:datum_service:info)
+        max without (datum_service) (label_replace((controller_runtime:reconcile_stalled:warning unless on (cluster, namespace, deployment) deployment:datum_service:info) * on (cluster, namespace) group_left (datum_service, team) namespace:datum_service:info, "service", "$1", "datum_service", "(.*)"))
+        or
+        (controller_runtime:reconcile_stalled:warning unless on (cluster, namespace, deployment) deployment:datum_service:info unless on (cluster, namespace) namespace:datum_service:info)
       for: 15m
       keep_firing_for: 30m
       labels:
@@ -46,7 +48,9 @@ spec:
       expr: |
         max without (datum_service) (label_replace(controller_runtime:reconcile_stalled:critical * on (cluster, namespace, deployment) group_left (datum_service, team) deployment:datum_service:info, "service", "$1", "datum_service", "(.*)"))
         or
-        (controller_runtime:reconcile_stalled:critical unless on (cluster, namespace, deployment) deployment:datum_service:info)
+        max without (datum_service) (label_replace((controller_runtime:reconcile_stalled:critical unless on (cluster, namespace, deployment) deployment:datum_service:info) * on (cluster, namespace) group_left (datum_service, team) namespace:datum_service:info, "service", "$1", "datum_service", "(.*)"))
+        or
+        (controller_runtime:reconcile_stalled:critical unless on (cluster, namespace, deployment) deployment:datum_service:info unless on (cluster, namespace) namespace:datum_service:info)
       for: 10m
       keep_firing_for: 30m
       labels:
@@ -65,7 +69,9 @@ spec:
       expr: |
         max without (datum_service) (label_replace(controller_runtime:reconcile_error_floor:24h * on (cluster, namespace, deployment) group_left (datum_service, team) deployment:datum_service:info, "service", "$1", "datum_service", "(.*)"))
         or
-        (controller_runtime:reconcile_error_floor:24h unless on (cluster, namespace, deployment) deployment:datum_service:info)
+        max without (datum_service) (label_replace((controller_runtime:reconcile_error_floor:24h unless on (cluster, namespace, deployment) deployment:datum_service:info) * on (cluster, namespace) group_left (datum_service, team) namespace:datum_service:info, "service", "$1", "datum_service", "(.*)"))
+        or
+        (controller_runtime:reconcile_error_floor:24h unless on (cluster, namespace, deployment) deployment:datum_service:info unless on (cluster, namespace) namespace:datum_service:info)
       labels:
         severity: warning
       annotations:

docs/runbooks/controller-health.md

@@ -21,8 +21,10 @@ assuming the problem is in NSO.
 
 An alert carries the `service` and `team` that own its deployment when the
 platform's owner map, `deployment:datum_service:info`, names that deployment.
-An alert without them belongs to a deployment the map does not name, or to a
-pod that no Deployment owns.
+When that map does not name the deployment, the alert takes the owner of its
+namespace from `namespace:datum_service:info`. An alert without either belongs
+to a deployment and a namespace that neither map names, or to a pod that no
+Deployment owns.
 
 ## Alert timing
 

test/prometheus-rules/controllers/controller-health-rules.yaml

@@ -22,7 +22,9 @@ groups:
     expr: |
       max without (datum_service) (label_replace(controller_runtime:reconcile_stalled:warning * on (cluster, namespace, deployment) group_left (datum_service, team) deployment:datum_service:info, "service", "$1", "datum_service", "(.*)"))
       or
-      (controller_runtime:reconcile_stalled:warning unless on (cluster, namespace, deployment) deployment:datum_service:info)
+      max without (datum_service) (label_replace((controller_runtime:reconcile_stalled:warning unless on (cluster, namespace, deployment) deployment:datum_service:info) * on (cluster, namespace) group_left (datum_service, team) namespace:datum_service:info, "service", "$1", "datum_service", "(.*)"))
+      or
+      (controller_runtime:reconcile_stalled:warning unless on (cluster, namespace, deployment) deployment:datum_service:info unless on (cluster, namespace) namespace:datum_service:info)
     for: 15m
     keep_firing_for: 30m
     labels:
@@ -40,7 +42,9 @@ groups:
     expr: |
       max without (datum_service) (label_replace(controller_runtime:reconcile_stalled:critical * on (cluster, namespace, deployment) group_left (datum_service, team) deployment:datum_service:info, "service", "$1", "datum_service", "(.*)"))
       or
-      (controller_runtime:reconcile_stalled:critical unless on (cluster, namespace, deployment) deployment:datum_service:info)
+      max without (datum_service) (label_replace((controller_runtime:reconcile_stalled:critical unless on (cluster, namespace, deployment) deployment:datum_service:info) * on (cluster, namespace) group_left (datum_service, team) namespace:datum_service:info, "service", "$1", "datum_service", "(.*)"))
+      or
+      (controller_runtime:reconcile_stalled:critical unless on (cluster, namespace, deployment) deployment:datum_service:info unless on (cluster, namespace) namespace:datum_service:info)
     for: 10m
     keep_firing_for: 30m
     labels:
@@ -59,7 +63,9 @@ groups:
     expr: |
       max without (datum_service) (label_replace(controller_runtime:reconcile_error_floor:24h * on (cluster, namespace, deployment) group_left (datum_service, team) deployment:datum_service:info, "service", "$1", "datum_service", "(.*)"))
       or
-      (controller_runtime:reconcile_error_floor:24h unless on (cluster, namespace, deployment) deployment:datum_service:info)
+      max without (datum_service) (label_replace((controller_runtime:reconcile_error_floor:24h unless on (cluster, namespace, deployment) deployment:datum_service:info) * on (cluster, namespace) group_left (datum_service, team) namespace:datum_service:info, "service", "$1", "datum_service", "(.*)"))
+      or
+      (controller_runtime:reconcile_error_floor:24h unless on (cluster, namespace, deployment) deployment:datum_service:info unless on (cluster, namespace) namespace:datum_service:info)
     labels:
       severity: warning
     annotations:

test/prometheus-rules/controllers/controller-health-tests.yaml

@@ -357,6 +357,91 @@ tests:
           description: The 'gateway' controller in cluster 'prod', namespace 'nso-system' has returned reconcile errors in every hour for a day. A steady error floor is almost always a failure no retry can fix, or a wait, returned as a plain error. Find the objects in the controller logs and fix the controller or the data; see the runbook.
           runbook_url: https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/controller-health.md#controllerreconcileerrorfloor
 
+  # The namespace step: a deployment the owner map does not name takes its namespace's owner from
+  # namespace:datum_service:info, the platform's next step after the deployment. A deployment both maps name keeps
+  # the deployment's owner and fires once. A deployment named in neither map still fires, without an owner, so no
+  # alert is lost.
+  - interval: 1m
+    input_series:
+    - series: controller_runtime_reconcile_total{controller="gateway",result="error",cluster="prod",namespace="nso-system",pod="nso-controller-manager-7d9f8b6c4-x2k9p"}
+      values: 0+9x60
+    - series: controller_runtime_reconcile_total{controller="gateway",result="success",cluster="prod",namespace="nso-system",pod="nso-controller-manager-7d9f8b6c4-x2k9p"}
+      values: 0+1x60
+    - series: deployment:datum_service:info{cluster="prod",namespace="nso-system",deployment="nso-controller-manager",datum_service="network-services",team="platform"}
+      values: 1x60
+    - series: controller_runtime_reconcile_total{controller="gateway",result="error",cluster="prod",namespace="nso-system",pod="other-manager-5c6d7e8f9-q1w2e"}
+      values: 0+9x60
+    - series: controller_runtime_reconcile_total{controller="gateway",result="success",cluster="prod",namespace="nso-system",pod="other-manager-5c6d7e8f9-q1w2e"}
+      values: 0+1x60
+    - series: controller_runtime_reconcile_total{controller="gateway",result="error",cluster="prod",namespace="lonely-system",pod="lonely-manager-5c6d7e8f9-a1b2c"}
+      values: 0+9x60
+    - series: controller_runtime_reconcile_total{controller="gateway",result="success",cluster="prod",namespace="lonely-system",pod="lonely-manager-5c6d7e8f9-a1b2c"}
+      values: 0+1x60
+    - series: namespace:datum_service:info{cluster="prod",namespace="nso-system",datum_service="alb",team="delivery"}
+      values: 1x60
+    alert_rule_test:
+    - eval_time: 30m
+      alertname: ControllerReconcileErrorRatioCritical
+      exp_alerts:
+      - exp_labels:
+          severity: critical
+          controller: gateway
+          cluster: prod
+          namespace: nso-system
+          deployment: nso-controller-manager
+          service: network-services
+          team: platform
+        exp_annotations:
+          summary: Controller gateway in prod/nso-system/nso-controller-manager reconcile error ratio is 90% (threshold 50%) — reconciliation is stalled
+          description: More than 50% of the 'gateway' controller's reconcile attempts are failing in cluster 'prod', namespace 'nso-system' — it has effectively stopped applying changes. Treat as an active outage for anything this controller programs. Check controller logs for 'Reconciler error' entries. See ControllerReconcileErrorRatioHigh for initial diagnosis.
+          runbook_url: https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/controller-health.md#controllerreconcileerrorratiocritical
+      - exp_labels:
+          severity: critical
+          controller: gateway
+          cluster: prod
+          namespace: nso-system
+          deployment: other-manager
+          service: alb
+          team: delivery
+        exp_annotations:
+          summary: Controller gateway in prod/nso-system/other-manager reconcile error ratio is 90% (threshold 50%) — reconciliation is stalled
+          description: More than 50% of the 'gateway' controller's reconcile attempts are failing in cluster 'prod', namespace 'nso-system' — it has effectively stopped applying changes. Treat as an active outage for anything this controller programs. Check controller logs for 'Reconciler error' entries. See ControllerReconcileErrorRatioHigh for initial diagnosis.
+          runbook_url: https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/controller-health.md#controllerreconcileerrorratiocritical
+      - exp_labels:
+          severity: critical
+          controller: gateway
+          cluster: prod
+          namespace: lonely-system
+          deployment: lonely-manager
+        exp_annotations:
+          summary: Controller gateway in prod/lonely-system/lonely-manager reconcile error ratio is 90% (threshold 50%) — reconciliation is stalled
+          description: More than 50% of the 'gateway' controller's reconcile attempts are failing in cluster 'prod', namespace 'lonely-system' — it has effectively stopped applying changes. Treat as an active outage for anything this controller programs. Check controller logs for 'Reconciler error' entries. See ControllerReconcileErrorRatioHigh for initial diagnosis.
+          runbook_url: https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/controller-health.md#controllerreconcileerrorratiocritical
+
+  # The floor warning takes the namespace's owner the same way.
+  - interval: 5m
+    input_series:
+    - series: controller_runtime_reconcile_total{controller="gateway",result="error",cluster="prod",namespace="nso-system",pod="other-manager-5c6d7e8f9-q1w2e"}
+      values: 0+1x320
+    - series: namespace:datum_service:info{cluster="prod",namespace="nso-system",datum_service="alb",team="delivery"}
+      values: 1x320
+    alert_rule_test:
+    - eval_time: 25h
+      alertname: ControllerReconcileErrorFloor
+      exp_alerts:
+      - exp_labels:
+          severity: warning
+          controller: gateway
+          cluster: prod
+          namespace: nso-system
+          deployment: other-manager
+          service: alb
+          team: delivery
+        exp_annotations:
+          summary: Controller gateway in prod/nso-system/other-manager has had reconcile errors in each of the last 24 hours
+          description: The 'gateway' controller in cluster 'prod', namespace 'nso-system' has returned reconcile errors in every hour for a day. A steady error floor is almost always a failure no retry can fix, or a wait, returned as a plain error. Find the objects in the controller logs and fix the controller or the data; see the runbook.
+          runbook_url: https://github.com/datum-cloud/network-services-operator/blob/main/docs/runbooks/controller-health.md#controllerreconcileerrorfloor
+
   # An owner entry that appears while an alert fires gives the alert a new label set, so it is a new alert: the
   # owned one waits out its own `for`, and `keep_firing_for` holds the unowned one for 30 minutes, so both fire
   # for a while and neither gap nor page is lost.