From 626d55ad8e44a9771eace49a0dae8997bbb8f03a Mon Sep 17 00:00:00 2001 From: VioletCranberry Date: Mon, 21 Sep 2026 16:12:05 +0200 Subject: [PATCH 1/2] fix(suse-observability): make metrics-processing dashboard usable for sizing The churn, slow-insert and slow-query panels pinned pod_name=suse-observability-victoria-metrics-0-0, so HA instance 1 stayed invisible even after #223 made it report metrics. Break them down by pod_name across all VictoriaMetrics pods instead. Both CPU Throttling queries returned no series: they divide by container_cpu_elapsed_periods, which is 0 when a container has no CPU limit, so the ratio was NaN. Collapse them into one query grouped on container, which covers all four components rather than splitting across kube_app_component and kube_app_name, and guard the denominator. Throttling cannot exist without a CPU limit, so add a CPU Pressure panel on container_cpu_partial_stall, which reports contention regardless. That is the signal needed when comparing CPU requests. Add request and limit reference lines to both CPU usage panels so headroom is readable. --- .../dashboards/metrics-processing.sty | 159 ++++++++++++++---- stackpacks/suse-observability/stackpack.yaml | 2 +- 2 files changed, 123 insertions(+), 38 deletions(-) diff --git a/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty b/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty index 4fc686c..267ddd1 100644 --- a/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty +++ b/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty @@ -37,14 +37,15 @@ nodes: plugin: kind: PrometheusTimeSeriesQuery spec: - query: max(rate(stackstate_vm_slow_row_inserts_total{cluster_name="${cluster}", + query: sum by + (pod_name)(rate(stackstate_vm_slow_row_inserts_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}]) - / - rate(stackstate_vm_rows_added_to_storage_total{cluster_name="${cluster}", + pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + / sum by + (pod_name)(rate(stackstate_vm_rows_added_to_storage_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}])) - alias: Percentage of slow inserts + pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + alias: ${pod_name} display: name: VictoriaMetrics - Slow inserts description: "Percentage of inserts into VictoriaMetrics that are considered slow." @@ -75,10 +76,11 @@ nodes: plugin: kind: PrometheusTimeSeriesQuery spec: - query: sum(rate(stackstate_vm_slow_queries_total{cluster_name="${cluster}", + query: sum by + (pod_name)(rate(stackstate_vm_slow_queries_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}])) - alias: Slow queries rate + pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + alias: ${pod_name} display: name: VictoriaMetrics - Slow queries rate description: "Rate of slow queries executed against VictoriaMetrics." @@ -109,19 +111,21 @@ nodes: plugin: kind: PrometheusTimeSeriesQuery spec: - query: sum(increase(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", + query: sum by + (pod_name)(increase(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name="suse-observability-victoria-metrics-0-0"}[24h])) - alias: Time series created over 24 hrs + pod_name=~".*-victoria-metrics-.*"}[24h])) + alias: ${pod_name} over 24 hrs - kind: TimeSeriesQuery spec: plugin: kind: PrometheusTimeSeriesQuery spec: - query: sum(rate(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", + query: sum by + (pod_name)(rate(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}])) - alias: Time series created / second + pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + alias: ${pod_name} / second display: name: VictoriaMetrics - Churn rate description: "Rate of new time series creation in VictoriaMetrics." @@ -198,34 +202,23 @@ nodes: plugin: kind: PrometheusTimeSeriesQuery spec: - query: 100 * sum by (kube_app_component) - (container_cpu_throttled_periods{cluster_name="${cluster}", - namespace="${namespace}", - kube_app_component=~"receiver|receiver-base|vmagent"}) / sum - by (kube_app_component) - (container_cpu_elapsed_periods{cluster_name="${cluster}", - namespace="${namespace}", - kube_app_component=~"receiver|receiver-base|vmagent"}) - alias: ${kube_app_component} - - kind: TimeSeriesQuery - spec: - plugin: - kind: PrometheusTimeSeriesQuery - spec: - query: 100 * sum by (kube_app_name) + query: 100 * sum by (container) (container_cpu_throttled_periods{cluster_name="${cluster}", namespace="${namespace}", - kube_app_name=~"opentelemetry-collector|victoria-metrics-\\d"}) - / sum by (kube_app_name) + container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) + / (sum by (container) (container_cpu_elapsed_periods{cluster_name="${cluster}", namespace="${namespace}", - kube_app_name=~"opentelemetry-collector|victoria-metrics-\\d"}) - alias: ${kube_app_name} + container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) + > 0) + alias: ${container} display: name: CPU Throttling description: CPU throttling for the main pods in the data ingestion pipeline. High throttling can lead to delays in data processing, a profile - upgrade or CPU limit increase can resolve the problem. + upgrade or CPU limit increase can resolve the problem. A container + with no CPU limit is never throttled and reports nothing here, so + read this alongside CPU Pressure. 1LYJbBBKSUMFsD1dbNDvi: spec: plugin: @@ -270,9 +263,31 @@ nodes: kube_app_name="opentelemetry-collector"}[${__interval}])) / 1000000000 alias: ${kube_app_name} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (container)(kubernetes_state_container_cpu_requested{cluster_name="${cluster}", + namespace="${namespace}", + container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) + alias: ${container} request + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (container)(kubernetes_state_container_cpu_limit{cluster_name="${cluster}", + namespace="${namespace}", + container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) + alias: ${container} limit display: name: CPU Usage Processing - description: "CPU usage of receiver, vmagent, and OpenTelemetry collector pods." + description: "CPU usage of receiver, vmagent, and OpenTelemetry collector + pods, against their configured request and limit. A missing limit line + means no CPU limit is set." jGL_6sXlItqnyvKC2NU_D: spec: plugin: @@ -306,9 +321,30 @@ nodes: pod_name=~"suse-observability-(victoria-metrics).*"}[${__interval}])) / 1000000000 alias: ${pod_name} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(kubernetes_state_container_cpu_requested{cluster_name="${cluster}", + namespace="${namespace}", + container=~"victoria-metrics-\\d-server"}) + alias: ${pod_name} request + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(kubernetes_state_container_cpu_limit{cluster_name="${cluster}", + namespace="${namespace}", + container=~"victoria-metrics-\\d-server"}) + alias: ${pod_name} limit display: name: CPU Usage VictoriaMetrics - description: "CPU usage of VictoriaMetrics pods." + description: "CPU usage of VictoriaMetrics pods, against their configured + request and limit. A missing limit line means no CPU limit is set." LENPxpypcWV-l3jYWsgPA: spec: plugin: @@ -348,6 +384,49 @@ nodes: display: name: Volume usage description: "Percentage of persistent volume capacity used by VictoriaMetrics." + cpuPressurePanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 3 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (container)(max_over_time(container_cpu_partial_stall{cluster_name="${cluster}", + namespace="${namespace}", + container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}[${__interval}])) + / 1000000000 + alias: ${container} + display: + name: CPU Pressure + description: >- + Time these containers spent stalled waiting for CPU, from kernel + pressure stall information. + + Unlike CPU Throttling this reports contention even when no CPU limit + is set, so it is the signal to use when comparing CPU requests. Read + it as a trend: rising stall alongside flat throughput means the + workload is CPU starved. Ao0rRQFkI3j5Va7erPJP3: spec: plugin: @@ -418,6 +497,12 @@ nodes: height: 2 content: $ref: "#/spec/panels/jGL_6sXlItqnyvKC2NU_D" + - x: 0 + y: 6 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/cpuPressurePanelMetricsPath" - x: 8 y: 0 width: 8 diff --git a/stackpacks/suse-observability/stackpack.yaml b/stackpacks/suse-observability/stackpack.yaml index 3be266c..e55cda5 100644 --- a/stackpacks/suse-observability/stackpack.yaml +++ b/stackpacks/suse-observability/stackpack.yaml @@ -1,5 +1,5 @@ name: suse-observability -version: "0.0.7" +version: "0.0.8" schemaVersion: "2.0" displayName: "SUSE Observability" categories: [ "SUSE Observability" ] From 9c9dfa28e87a7fd6d9f2fca945a8b2645654e6c6 Mon Sep 17 00:00:00 2001 From: VioletCranberry Date: Tue, 22 Sep 2026 11:09:13 +0200 Subject: [PATCH 2/2] feat(suse-observability): add queue panels and tighten selectors Add the queue signals the sizing work needs, with no existing panel: VictoriaMetrics pending rows split by storage and indexdb, vmagent persistent-queue write time alongside pending bytes, collector metrics export queue occupancy, and receiver memory-limiter saturation. Collector queue is plotted as occupancy against queue_capacity rather than raw depth, since capacity differs per exporter and profile. The receiver has no queue, so its memory limiter stands in: reaching the limit is what makes it reject incoming data. Scope every container selector with kube_app_instance, since container names alone are not specific enough. kube_app_name cannot serve here - receiver and vmagent report it as suse-observability while the collector and VictoriaMetrics report their own names. Anchor the pod_name selectors on the suse-observability prefix, which the chart fixes via fullnameOverride rather than deriving from the release name. --- .../dashboards/metrics-processing.sty | 238 ++++++++++++++++-- 1 file changed, 223 insertions(+), 15 deletions(-) diff --git a/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty b/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty index 267ddd1..ba72408 100644 --- a/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty +++ b/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty @@ -40,11 +40,11 @@ nodes: query: sum by (pod_name)(rate(stackstate_vm_slow_row_inserts_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) / sum by (pod_name)(rate(stackstate_vm_rows_added_to_storage_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) alias: ${pod_name} display: name: VictoriaMetrics - Slow inserts @@ -79,7 +79,7 @@ nodes: query: sum by (pod_name)(rate(stackstate_vm_slow_queries_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) alias: ${pod_name} display: name: VictoriaMetrics - Slow queries rate @@ -114,7 +114,7 @@ nodes: query: sum by (pod_name)(increase(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name=~".*-victoria-metrics-.*"}[24h])) + pod_name=~"suse-observability-victoria-metrics-.*"}[24h])) alias: ${pod_name} over 24 hrs - kind: TimeSeriesQuery spec: @@ -124,7 +124,7 @@ nodes: query: sum by (pod_name)(rate(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", namespace="${namespace}", - pod_name=~".*-victoria-metrics-.*"}[${__rate_interval}])) + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) alias: ${pod_name} / second display: name: VictoriaMetrics - Churn rate @@ -152,7 +152,7 @@ nodes: kind: PrometheusTimeSeriesQuery spec: query: sum by - (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~".*-vmagent-.*", + (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~"suse-observability-vmagent-.*", cluster_name="${cluster}", namespace="${namespace}"}[1h])) alias: vmagent - kind: TimeSeriesQuery @@ -161,7 +161,7 @@ nodes: kind: PrometheusTimeSeriesQuery spec: query: sum by - (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~".*-victoria-metrics-.*", + (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~"suse-observability-victoria-metrics-.*", cluster_name="${cluster}", namespace="${namespace}"}[1h])) alias: VictoriaMetrics display: @@ -205,11 +205,11 @@ nodes: query: 100 * sum by (container) (container_cpu_throttled_periods{cluster_name="${cluster}", namespace="${namespace}", - container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) / (sum by (container) (container_cpu_elapsed_periods{cluster_name="${cluster}", namespace="${namespace}", - container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) > 0) alias: ${container} display: @@ -271,7 +271,7 @@ nodes: query: sum by (container)(kubernetes_state_container_cpu_requested{cluster_name="${cluster}", namespace="${namespace}", - container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) alias: ${container} request - kind: TimeSeriesQuery spec: @@ -281,7 +281,7 @@ nodes: query: sum by (container)(kubernetes_state_container_cpu_limit{cluster_name="${cluster}", namespace="${namespace}", - container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) alias: ${container} limit display: name: CPU Usage Processing @@ -318,7 +318,7 @@ nodes: query: sum by (cluster_name, namespace, pod_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", namespace="${namespace}", - pod_name=~"suse-observability-(victoria-metrics).*"}[${__interval}])) + pod_name=~"suse-observability-victoria-metrics-.*"}[${__interval}])) / 1000000000 alias: ${pod_name} - kind: TimeSeriesQuery @@ -329,7 +329,7 @@ nodes: query: sum by (pod_name)(kubernetes_state_container_cpu_requested{cluster_name="${cluster}", namespace="${namespace}", - container=~"victoria-metrics-\\d-server"}) + kube_app_instance="suse-observability", container=~"victoria-metrics-\\d-server"}) alias: ${pod_name} request - kind: TimeSeriesQuery spec: @@ -339,7 +339,7 @@ nodes: query: sum by (pod_name)(kubernetes_state_container_cpu_limit{cluster_name="${cluster}", namespace="${namespace}", - container=~"victoria-metrics-\\d-server"}) + kube_app_instance="suse-observability", container=~"victoria-metrics-\\d-server"}) alias: ${pod_name} limit display: name: CPU Usage VictoriaMetrics @@ -414,7 +414,7 @@ nodes: query: sum by (container)(max_over_time(container_cpu_partial_stall{cluster_name="${cluster}", namespace="${namespace}", - container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}[${__interval}])) + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}[${__interval}])) / 1000000000 alias: ${container} display: @@ -427,6 +427,190 @@ nodes: is set, so it is the signal to use when comparing CPU requests. Read it as a trend: rising stall alongside flat throughput means the workload is CPU starved. + vmPendingRowsPanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (pod_name, + type)(stackstate_vm_pending_rows{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}) + alias: ${pod_name} ${type} + display: + name: VictoriaMetrics - Pending rows + description: >- + Rows buffered in memory before VictoriaMetrics has written them to + storage, split by storage and indexdb. + + A steady baseline is normal. Sustained growth means ingestion is + arriving faster than it can be persisted, and is the first place + backpressure appears in the metrics path. + vmagentQueueWritePanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 3 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(rate(stackstate_vm_persistentqueue_write_duration_seconds_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-vmagent-.*"}[${__rate_interval}])) + alias: ${pod_name} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(stackstate_vmagent_remotewrite_pending_data_bytes{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-vmagent-.*"}) / 1000000 + alias: ${pod_name} pending MB + display: + name: vmagent - Write queue + description: >- + Seconds per second that vmagent spent writing to its on-disk + persistent queue, alongside the megabytes currently pending. + + Both sitting at zero means remote write is keeping up. Writing time + above zero means vmagent is spilling to disk because the metric store + is not accepting data fast enough. + collectorQueueOccupancyPanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: + - color: "#FF7F00" + value: 0.8 + - color: "#E31A1C" + value: 0.9 + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (pod_name, + exporter)(stackstate_otelcol_exporter_queue_size{cluster_name="${cluster}", + namespace="${namespace}", data_type="metrics"}) / sum by + (pod_name, + exporter)(stackstate_otelcol_exporter_queue_capacity{cluster_name="${cluster}", + namespace="${namespace}", data_type="metrics"}) + alias: ${exporter} + display: + name: OTel Collector - Metrics export queue occupancy + description: >- + How full the collector's metric export queue is, as a fraction of its + configured capacity. + + Occupancy is the useful form rather than raw depth, because capacity + differs per exporter and per profile. A queue at capacity drops or + refuses data rather than buffering it further. + receiverBackpressurePanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: + - color: "#FF7F00" + value: 0.8 + - color: "#E31A1C" + value: 0.95 + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(stackstate_stackstate_receiver_memory_limiter_claimed_size{cluster_name="${cluster}", + namespace="${namespace}"}) / sum by + (pod_name)(stackstate_stackstate_receiver_memory_limiter_limit{cluster_name="${cluster}", + namespace="${namespace}"}) + alias: ${pod_name} + display: + name: Receiver - Memory limiter saturation + description: >- + Share of the receiver's memory limiter budget currently claimed. + + The receiver has no queue to measure; this is its backpressure + equivalent, because reaching the limit is what makes it start + rejecting incoming data. Ao0rRQFkI3j5Va7erPJP3: spec: plugin: @@ -503,6 +687,30 @@ nodes: height: 2 content: $ref: "#/spec/panels/cpuPressurePanelMetricsPath" + - x: 8 + y: 6 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/vmPendingRowsPanelMetricsPath" + - x: 16 + y: 6 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/vmagentQueueWritePanelMetricsPath" + - x: 0 + y: 8 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/collectorQueueOccupancyPanelMetricsPath" + - x: 8 + y: 8 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/receiverBackpressurePanelMetricsPath" - x: 8 y: 0 width: 8