From 96035392e4b60dba8775d56a3fd91fc941e789af Mon Sep 17 00:00:00 2001 From: VioletCranberry Date: Wed, 23 Sep 2026 12:20:56 +0200 Subject: [PATCH] fix(suse-observability): stop freshness and concurrency monitors misfiring The freshness monitor compared the instantaneous age of the newest sample against 30s. That age climbs to the scrape interval between every scrape, so healthy namespaces crossed the threshold each cycle and flapped. Take the minimum age over a one-minute window instead, which removes the scrape sawtooth and leaves the scrape-to-queryable delay. The aggregation has to sit outside the subquery; the metric backend rejects an aggregation inside it. Scope both concurrency monitors to VictoriaMetrics. vmagent exposes the same vm_concurrent_insert metrics for its own incoming writes, and on small deployments its capacity of 2 put it into DEVIATING on normal load. --- .../services/victoria-metrics/freshness.md.hbs | 8 ++++---- .../services/victoria-metrics/monitors.sty | 16 +++++++++------- stackpacks/suse-observability/stackpack.yaml | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs index c13d8e4..3db970d 100644 --- a/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs @@ -1,10 +1,10 @@ -The newest metric sample is older than the freshness objective, so dashboards and monitors are evaluating stale data. +Metrics are taking longer than the freshness objective to become queryable after they are scraped, so dashboards and monitors are evaluating late data. -Note this measures the age of arriving traffic, not end-to-end ingest latency. Check in order: +The value is the smallest sample age seen over the last minute, which removes the normal rise between scrapes and leaves the delay through the pipeline. Check in order: -- is anything still being written? `stackstate_vm_rows_inserted_total` should be rising - is vmagent holding data back? see the `vmagent - Write queue` panel - is the collector queue backing up? see the metrics export queue occupancy panel - is the receiver rejecting? see `Receiver - Memory limiter saturation` +- is VictoriaMetrics slow to persist? see `VictoriaMetrics - Pending rows` and `Insert concurrency saturated` -A jump here with healthy queues usually means scraping stopped rather than that the path is slow. +If scraping has stopped entirely this monitor fires only after about 90 seconds, and it goes quiet once the series expire. `Receiver - No data received` covers a full stop. diff --git a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty index e7f615c..c7b6d5b 100644 --- a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty +++ b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty @@ -267,8 +267,8 @@ nodes: arguments: metric: query: |- - sum(avg_over_time(stackstate_vm_concurrent_insert_current[1m])) by (cluster_name, namespace, pod_name) - / sum(stackstate_vm_concurrent_insert_capacity) by (cluster_name, namespace, pod_name) + sum(avg_over_time(stackstate_vm_concurrent_insert_current{kube_app_name=~"victoria-metrics-.*"}[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_insert_capacity{kube_app_name=~"victoria-metrics-.*"}) by (cluster_name, namespace, pod_name) unit: "percentunit" aliasTemplate: "Insert concurrency in use" comparator: GTE @@ -291,8 +291,8 @@ nodes: arguments: metric: query: |- - sum(avg_over_time(stackstate_vm_concurrent_select_current[1m])) by (cluster_name, namespace, pod_name) - / sum(stackstate_vm_concurrent_select_capacity) by (cluster_name, namespace, pod_name) + sum(avg_over_time(stackstate_vm_concurrent_select_current{kube_app_name=~"victoria-metrics-.*"}[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_select_capacity{kube_app_name=~"victoria-metrics-.*"}) by (cluster_name, namespace, pod_name) unit: "percentunit" aliasTemplate: "Select concurrency in use" comparator: GTE @@ -306,7 +306,7 @@ nodes: - _type: "Monitor" name: "VictoriaMetrics - Metrics are not fresh" - description: "Monitor that triggers when the newest metric sample is older than the freshness objective" + description: "Monitor that triggers when metrics take longer than the freshness objective to become queryable after they are scraped" tags: - victoria-metrics - suse-observability @@ -315,9 +315,11 @@ nodes: arguments: metric: query: |- - time() - max(timestamp(stackstate_vm_rows_inserted_total)) by (cluster_name, namespace, pod_name) + max by (cluster_name, namespace, pod_name) ( + min_over_time((time() - timestamp(stackstate_vm_rows_inserted_total{type="promremotewrite"}))[1m:5s]) + ) unit: "s" - aliasTemplate: "Age of the newest sample" + aliasTemplate: "Scrape to queryable delay" comparator: GT threshold: 30 failureState: "DEVIATING" diff --git a/stackpacks/suse-observability/stackpack.yaml b/stackpacks/suse-observability/stackpack.yaml index 32d97de..ac929b9 100644 --- a/stackpacks/suse-observability/stackpack.yaml +++ b/stackpacks/suse-observability/stackpack.yaml @@ -1,5 +1,5 @@ name: suse-observability -version: "0.0.9" +version: "0.0.10" schemaVersion: "2.0" displayName: "SUSE Observability" categories: [ "SUSE Observability" ]