diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs index c13d8e4..3db970d 100644 --- a/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs @@ -1,10 +1,10 @@ -The newest metric sample is older than the freshness objective, so dashboards and monitors are evaluating stale data. +Metrics are taking longer than the freshness objective to become queryable after they are scraped, so dashboards and monitors are evaluating late data. -Note this measures the age of arriving traffic, not end-to-end ingest latency. Check in order: +The value is the smallest sample age seen over the last minute, which removes the normal rise between scrapes and leaves the delay through the pipeline. Check in order: -- is anything still being written? `stackstate_vm_rows_inserted_total` should be rising - is vmagent holding data back? see the `vmagent - Write queue` panel - is the collector queue backing up? see the metrics export queue occupancy panel - is the receiver rejecting? see `Receiver - Memory limiter saturation` +- is VictoriaMetrics slow to persist? see `VictoriaMetrics - Pending rows` and `Insert concurrency saturated` -A jump here with healthy queues usually means scraping stopped rather than that the path is slow. +If scraping has stopped entirely this monitor fires only after about 90 seconds, and it goes quiet once the series expire. `Receiver - No data received` covers a full stop. diff --git a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty index e7f615c..c7b6d5b 100644 --- a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty +++ b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty @@ -267,8 +267,8 @@ nodes: arguments: metric: query: |- - sum(avg_over_time(stackstate_vm_concurrent_insert_current[1m])) by (cluster_name, namespace, pod_name) - / sum(stackstate_vm_concurrent_insert_capacity) by (cluster_name, namespace, pod_name) + sum(avg_over_time(stackstate_vm_concurrent_insert_current{kube_app_name=~"victoria-metrics-.*"}[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_insert_capacity{kube_app_name=~"victoria-metrics-.*"}) by (cluster_name, namespace, pod_name) unit: "percentunit" aliasTemplate: "Insert concurrency in use" comparator: GTE @@ -291,8 +291,8 @@ nodes: arguments: metric: query: |- - sum(avg_over_time(stackstate_vm_concurrent_select_current[1m])) by (cluster_name, namespace, pod_name) - / sum(stackstate_vm_concurrent_select_capacity) by (cluster_name, namespace, pod_name) + sum(avg_over_time(stackstate_vm_concurrent_select_current{kube_app_name=~"victoria-metrics-.*"}[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_select_capacity{kube_app_name=~"victoria-metrics-.*"}) by (cluster_name, namespace, pod_name) unit: "percentunit" aliasTemplate: "Select concurrency in use" comparator: GTE @@ -306,7 +306,7 @@ nodes: - _type: "Monitor" name: "VictoriaMetrics - Metrics are not fresh" - description: "Monitor that triggers when the newest metric sample is older than the freshness objective" + description: "Monitor that triggers when metrics take longer than the freshness objective to become queryable after they are scraped" tags: - victoria-metrics - suse-observability @@ -315,9 +315,11 @@ nodes: arguments: metric: query: |- - time() - max(timestamp(stackstate_vm_rows_inserted_total)) by (cluster_name, namespace, pod_name) + max by (cluster_name, namespace, pod_name) ( + min_over_time((time() - timestamp(stackstate_vm_rows_inserted_total{type="promremotewrite"}))[1m:5s]) + ) unit: "s" - aliasTemplate: "Age of the newest sample" + aliasTemplate: "Scrape to queryable delay" comparator: GT threshold: 30 failureState: "DEVIATING" diff --git a/stackpacks/suse-observability/stackpack.yaml b/stackpacks/suse-observability/stackpack.yaml index 32d97de..ac929b9 100644 --- a/stackpacks/suse-observability/stackpack.yaml +++ b/stackpacks/suse-observability/stackpack.yaml @@ -1,5 +1,5 @@ name: suse-observability -version: "0.0.9" +version: "0.0.10" schemaVersion: "2.0" displayName: "SUSE Observability" categories: [ "SUSE Observability" ]