From d59ba674b8541deba9d6ff385ea4e8189dec6306 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=EA=B0=95=EA=B2=BD=ED=98=84?=
 <108459576+kangkyunghyun@users.noreply.github.com>
Date: Tue, 11 Aug 2026 12:54:31 +0900
Subject: [PATCH] fix(dashboard): restore pipeline health metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 강경현 <108459576+kangkyunghyun@users.noreply.github.com>
---
 .../files/dashboard-pipeline-health.yaml      |  2 +-
 .../tests/init_dashboards_configmap_test.yaml |  9 ++++++
 .../dashboard-pipeline-health.yaml            |  2 +-
 docker-compose/otel-collector/config.yaml     | 30 +++++++++++++++++--
 4 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/charts/observability-stack/files/dashboard-pipeline-health.yaml b/charts/observability-stack/files/dashboard-pipeline-health.yaml
index 2c611cb2..dbb2ce63 100644
--- a/charts/observability-stack/files/dashboard-pipeline-health.yaml
+++ b/charts/observability-stack/files/dashboard-pipeline-health.yaml
@@ -36,7 +36,7 @@ panels:
 
   - id: pipeline-otel-collector-memory
     title: "OTel Collector Memory (bytes)"
-    query: "otelcol_process_memory_rss_bytes"
+    query: "otelcol_process_memory_rss"
     chartType: line
 
   # --- Row 4: OTel Collector CPU & Uptime ---
diff --git a/charts/observability-stack/tests/init_dashboards_configmap_test.yaml b/charts/observability-stack/tests/init_dashboards_configmap_test.yaml
index 614ce69d..930aef9b 100644
--- a/charts/observability-stack/tests/init_dashboards_configmap_test.yaml
+++ b/charts/observability-stack/tests/init_dashboards_configmap_test.yaml
@@ -22,3 +22,12 @@ tests:
       - equal:
           path: metadata.annotations["helm.sh/hook"]
           value: post-install,post-upgrade
+
+  - it: should use the collector RSS metric exposed by the OTel Collector
+    asserts:
+      - matchRegex:
+          path: data["dashboard-pipeline-health.yaml"]
+          pattern: 'query: "otelcol_process_memory_rss"'
+      - notMatchRegex:
+          path: data["dashboard-pipeline-health.yaml"]
+          pattern: "otelcol_process_memory_rss_bytes"
diff --git a/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml b/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml
index 2c611cb2..dbb2ce63 100644
--- a/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml
+++ b/docker-compose/opensearch-dashboards/dashboard-pipeline-health.yaml
@@ -36,7 +36,7 @@ panels:
 
   - id: pipeline-otel-collector-memory
     title: "OTel Collector Memory (bytes)"
-    query: "otelcol_process_memory_rss_bytes"
+    query: "otelcol_process_memory_rss"
     chartType: line
 
   # --- Row 4: OTel Collector CPU & Uptime ---
diff --git a/docker-compose/otel-collector/config.yaml b/docker-compose/otel-collector/config.yaml
index 87a848cf..808953e0 100644
--- a/docker-compose/otel-collector/config.yaml
+++ b/docker-compose/otel-collector/config.yaml
@@ -56,6 +56,32 @@ receivers:
             - target_label: service.name
               replacement: frontend-proxy
 
+  # Cortex internal metrics for the Pipeline Health dashboard.
+  prometheus/cortex:
+    config:
+      scrape_configs:
+        - job_name: cortex
+          scrape_interval: 15s
+          metrics_path: /metrics
+          static_configs:
+            - targets: ["prometheus:9090"]
+          relabel_configs:
+            - target_label: service.name
+              replacement: cortex
+
+  # Data Prepper pipeline metrics for the Pipeline Health dashboard.
+  prometheus/data-prepper:
+    config:
+      scrape_configs:
+        - job_name: data-prepper-pipelines
+          scrape_interval: 15s
+          metrics_path: /metrics/prometheus
+          static_configs:
+            - targets: ["data-prepper:4900"]
+          relabel_configs:
+            - target_label: service.name
+              replacement: data-prepper
+
 processors:
   # Memory limiter prevents OOM by dropping data when memory usage is high
   # Critical for stability under load
@@ -171,9 +197,9 @@ service:
       processors: [resourcedetection, memory_limiter, gen_ai_normalizer, transform, batch]
       exporters: [otlp/opensearch, debug]
 
-    # Metrics pipeline: OTLP + self-scrape + envoy scrape -> processing -> Cortex
+    # Metrics pipeline: OTLP + component scrapes -> processing -> Cortex
     metrics:
-      receivers: [otlp, prometheus/self, prometheus/envoy]
+      receivers: [otlp, prometheus/self, prometheus/envoy, prometheus/cortex, prometheus/data-prepper]
       processors: [resourcedetection, memory_limiter, batch]
       exporters: [prometheusremotewrite/cortex, debug]