From b56fff6568d9f8817d2f01ae65ed457f9f9c09d3 Mon Sep 17 00:00:00 2001 From: VioletCranberry Date: Tue, 22 Sep 2026 13:15:24 +0200 Subject: [PATCH] feat(suse-observability): add monitors for uncovered metrics-path failures Six failure modes could occur while every existing monitor stayed green: VictoriaMetrics switching to read-only, insert and select concurrency reaching the CPU-derived limit, vmagent discarding samples on remote write, metrics going stale, and monitors overrunning their interval. Thresholds come from observed baselines rather than guesses. Monitor execution time triggers above 60s because the slowest monitor currently averages 25s. Concurrency triggers at 90% of capacity. Freshness uses 30s to match the topology SLO. Skip a pending-rows growth monitor: the baseline is environment-dependent (~22k storage, ~600 indexdb on one live namespace), so any absolute threshold either pages on normal behaviour or never fires. The dashboard panel covers it. Refs #230 --- .../checks/monitors-execution-time-pod.md.hbs | 9 ++ .../victoria-metrics/freshness.md.hbs | 10 ++ .../insert-concurrency.md.hbs | 5 + .../select-concurrency.md.hbs | 5 + .../victoria-metrics/storage-read-only.md.hbs | 8 ++ .../remotewrite-samples-dropped.md.hbs | 9 ++ .../settings/services/checks/monitors.sty | 24 +++++ .../services/victoria-metrics/monitors.sty | 94 +++++++++++++++++++ .../settings/services/vmagent/monitors.sty | 24 ++++- stackpacks/suse-observability/stackpack.yaml | 2 +- 10 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs diff --git a/stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs b/stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs new file mode 100644 index 0000000..7e5e8eb --- /dev/null +++ b/stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs @@ -0,0 +1,9 @@ +This monitor takes longer on average to evaluate than a one minute interval, so runs cannot reliably complete before the next one is due, and results arrive late or are skipped. + +Check: + +- the monitor's own query - long ranges over many series are the usual cause +- `Select concurrency saturated` on VictoriaMetrics, since a saturated query path slows every monitor at once +- whether the `checks` pod (or `server`, in non-HA) is CPU starved, on the *Metrics processing* dashboard + +Either simplify the query or lengthen the monitor's interval so it matches how long it actually takes. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs new file mode 100644 index 0000000..c13d8e4 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs @@ -0,0 +1,10 @@ +The newest metric sample is older than the freshness objective, so dashboards and monitors are evaluating stale data. + +Note this measures the age of arriving traffic, not end-to-end ingest latency. Check in order: + +- is anything still being written? `stackstate_vm_rows_inserted_total` should be rising +- is vmagent holding data back? see the `vmagent - Write queue` panel +- is the collector queue backing up? see the metrics export queue occupancy panel +- is the receiver rejecting? see `Receiver - Memory limiter saturation` + +A jump here with healthy queues usually means scraping stopped rather than that the path is slow. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs new file mode 100644 index 0000000..abc0865 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs @@ -0,0 +1,5 @@ +Concurrent inserts are at the limit VictoriaMetrics derives from the CPU it detects, so new writes queue behind in-flight ones. Sustained saturation shows up downstream as vmagent queueing and then as lost data. + +Check `stackstate_vm_concurrent_insert_limit_reached_total` to confirm writes are actually being held, and the `CPU Usage VictoriaMetrics` and `CPU Pressure` panels to see whether the pod is CPU starved. + +The limit scales with available CPU, so raising the CPU request or limit raises it. Reducing ingestion rate or series churn lowers demand for it. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs new file mode 100644 index 0000000..321f55f --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs @@ -0,0 +1,5 @@ +Concurrent queries are at the limit VictoriaMetrics derives from the CPU it detects. Further queries wait, and can time out. + +Check `stackstate_vm_concurrent_select_limit_reached_total` to confirm queries are being held, and `stackstate_vm_slow_queries_total` for expensive queries driving it. + +Usual causes are dashboards or monitors querying long ranges over many series. Narrow the heaviest queries, or raise the CPU allocation - the limit scales with detected CPU. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs new file mode 100644 index 0000000..25ff973 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs @@ -0,0 +1,8 @@ +VictoriaMetrics has switched to read-only mode and is no longer accepting writes. Metrics sent while this lasts are lost once the upstream queues fill. + +This is almost always free disk space falling below `-storage.minFreeDiskSpaceBytes`. Check: + +- the `Volume usage` panel on the *Metrics processing* dashboard, and the `Disk runs out of space` monitors +- `stackstate_vm_free_disk_space_bytes` against `stackstate_vm_free_disk_space_limit_bytes` for this pod + +To recover, free space or grow the volume. Reducing retention releases space only after the next merge, so it is not an immediate fix. diff --git a/stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs b/stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs new file mode 100644 index 0000000..c7f548f --- /dev/null +++ b/stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs @@ -0,0 +1,9 @@ +vmagent is discarding samples rather than delivering them. This is unrecoverable data loss: unlike a full queue, dropped samples are never retried. + +Check: + +- `stackstate_vmagent_remotewrite_pending_data_bytes` and the persistent queue - drops usually follow a queue that has been full for some time +- whether VictoriaMetrics is rejecting writes, via the `Rows rejected` and `Storage is read only` monitors +- `stackstate_vmagent_remotewrite_rate_limit_reached_total` if a remote-write rate limit is configured + +Fix the downstream cause rather than growing the queue; a larger queue only delays the same loss. diff --git a/stackpacks/suse-observability/settings/services/checks/monitors.sty b/stackpacks/suse-observability/settings/services/checks/monitors.sty index 4e2d906..fb0a935 100644 --- a/stackpacks/suse-observability/settings/services/checks/monitors.sty +++ b/stackpacks/suse-observability/settings/services/checks/monitors.sty @@ -24,3 +24,27 @@ nodes: intervalSeconds: 30 remediationHint: !include services/checks/monitors-failure-percentage-pod.md.hbs status: "ENABLED" + + - _type: "Monitor" + name: "Monitor execution time" + description: "Monitor that triggers when a monitor takes so long to evaluate that it cannot keep up with a one minute interval" + tags: + - monitors + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:monitors-execution-time-pod + arguments: + metric: + query: |- + sum by(cluster_name, namespace, pod_name, monitorName) (rate(stackstate_stackstate_monitor_latency_seconds_sum[10m])) + / sum by(cluster_name, namespace, pod_name, monitorName) (rate(stackstate_stackstate_monitor_latency_seconds_count[10m])) + unit: "s" + aliasTemplate: "Mean run duration" + comparator: GT + threshold: 60 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Monitor ${monitorName} execution time" + intervalSeconds: 60 + remediationHint: !include services/checks/monitors-execution-time-pod.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty index 8548d77..e7f615c 100644 --- a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty +++ b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty @@ -232,3 +232,97 @@ nodes: intervalSeconds: 90 remediationHint: !include services/victoria-metrics/labels-limit.md.hbs status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Storage is read only" + description: "Monitor that triggers when VictoriaMetrics has stopped accepting writes" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-storage-read-only + arguments: + metric: + query: |- + sum(stackstate_vm_storage_is_read_only) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Storage is read only" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Storage is read only" + intervalSeconds: 60 + remediationHint: !include services/victoria-metrics/storage-read-only.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Insert concurrency saturated" + description: "Monitor that triggers when concurrent inserts reach the limit VictoriaMetrics derives from its CPU allocation" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-insert-concurrency + arguments: + metric: + query: |- + sum(avg_over_time(stackstate_vm_concurrent_insert_current[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_insert_capacity) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Insert concurrency in use" + comparator: GTE + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Insert concurrency saturated" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/insert-concurrency.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Select concurrency saturated" + description: "Monitor that triggers when concurrent queries reach the limit VictoriaMetrics derives from its CPU allocation" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-select-concurrency + arguments: + metric: + query: |- + sum(avg_over_time(stackstate_vm_concurrent_select_current[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_select_capacity) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Select concurrency in use" + comparator: GTE + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Select concurrency saturated" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/select-concurrency.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Metrics are not fresh" + description: "Monitor that triggers when the newest metric sample is older than the freshness objective" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-freshness + arguments: + metric: + query: |- + time() - max(timestamp(stackstate_vm_rows_inserted_total)) by (cluster_name, namespace, pod_name) + unit: "s" + aliasTemplate: "Age of the newest sample" + comparator: GT + threshold: 30 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Metrics are not fresh" + intervalSeconds: 60 + remediationHint: !include services/victoria-metrics/freshness.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability/settings/services/vmagent/monitors.sty b/stackpacks/suse-observability/settings/services/vmagent/monitors.sty index d6e5ba6..58c7b7c 100644 --- a/stackpacks/suse-observability/settings/services/vmagent/monitors.sty +++ b/stackpacks/suse-observability/settings/services/vmagent/monitors.sty @@ -140,4 +140,26 @@ nodes: remediationHint: "Persistent queue writes for vmagent are saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput." - status: "ENABLED" \ No newline at end of file + status: "ENABLED" + - _type: "Monitor" + name: "Vmagent - Samples dropped on remote write" + description: "Monitor that triggers when vmagent discards samples instead of delivering them to the metric store" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-remotewrite-samples-dropped + arguments: + metric: + query: |- + sum(increase(stackstate_vmagent_remotewrite_samples_dropped_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Samples dropped" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Samples dropped on remote write" + intervalSeconds: 90 + remediationHint: !include services/vmagent/remotewrite-samples-dropped.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability/stackpack.yaml b/stackpacks/suse-observability/stackpack.yaml index e55cda5..32d97de 100644 --- a/stackpacks/suse-observability/stackpack.yaml +++ b/stackpacks/suse-observability/stackpack.yaml @@ -1,5 +1,5 @@ name: suse-observability -version: "0.0.8" +version: "0.0.9" schemaVersion: "2.0" displayName: "SUSE Observability" categories: [ "SUSE Observability" ]