From 72209dc370583cc26fe545f145c2f10eb8d118bf Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Tue, 27 Oct 2026 19:18:39 +0530 Subject: [PATCH 01/17] DLPX-100030 telegraf: add fluentd HTTP output for 3rd-party metric forwarding Add [[outputs.http]] stanza (telegraf.outputs.fluentd) that POSTs performance analytics metrics to the local fluentd HTTP source at http://localhost:9880/delphix.metrics.platform, where fluentd routes them to third-party monitoring systems (Splunk, Datadog) via existing rules. - namepass: cpu, diskio, net, zfs, estat_nfs, estat_iscsi - flush_interval: 60s (matches performanceMetricsResolution = MINUTE) - data_format: json - Gated on FLUENTD_ENABLED flag file (same pattern as INFLUXDB_ENABLED) Add perf_fluentd enable|disable CLI to manage the flag and restart Telegraf. Update delphix-telegraf-service to include the fluentd output when the flag is set, and extend the STORAGE_IO_INPUTS inclusion guard so estat_nfs/estat_iscsi are collected whenever either InfluxDB or fluentd is enabled. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- debian/rules | 2 +- telegraf/delphix-telegraf-service | 27 +++++++++++++++++-- telegraf/perf_fluentd | 45 +++++++++++++++++++++++++++++++ telegraf/telegraf.outputs.fluentd | 13 +++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 telegraf/perf_fluentd create mode 100644 telegraf/telegraf.outputs.fluentd diff --git a/debian/rules b/debian/rules index 00b38aa..d573e6c 100755 --- a/debian/rules +++ b/debian/rules @@ -23,7 +23,7 @@ override_dh_auto_install: dh_install build/cmd/* /usr/bin dh_install lib/* /usr/share/performance-diagnostics/lib dh_install bpf/* /usr/share/performance-diagnostics/bpf - dh_install telegraf/delphix-telegraf-service telegraf/perf_playbook /usr/bin + dh_install telegraf/delphix-telegraf-service telegraf/perf_playbook telegraf/perf_fluentd /usr/bin dh_install telegraf/delphix-telegraf.service /lib/systemd/system dh_install telegraf/telegraf* telegraf/*.sh /etc/telegraf dh_install influxdb/delphix-influxdb-service influxdb/delphix-influxdb-init influxdb/perf_influxdb /usr/bin diff --git a/telegraf/delphix-telegraf-service b/telegraf/delphix-telegraf-service index 3c9a4dc..b3a07f0 100755 --- a/telegraf/delphix-telegraf-service +++ b/telegraf/delphix-telegraf-service @@ -5,8 +5,10 @@ DCT_INPUTS=/etc/telegraf/telegraf.inputs.dct PLAYBOOK_INPUTS=/etc/telegraf/telegraf.inputs.playbook STORAGE_IO_INPUTS=/etc/telegraf/telegraf.inputs.storage_io INFLUXDB_OUTPUT=/etc/telegraf/telegraf.outputs.influxdb +FLUENTD_OUTPUT=/etc/telegraf/telegraf.outputs.fluentd PLAYBOOK_FLAG=/etc/telegraf/PLAYBOOK_ENABLED INFLUXDB_FLAG=/etc/telegraf/INFLUXDB_ENABLED +FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED TELEGRAF_CONFIG=/etc/telegraf/telegraf.conf TELEGRAF_CONFIG_DIR=/etc/telegraf/telegraf.d @@ -29,6 +31,10 @@ function influxdb_is_enabled() { [[ -f $INFLUXDB_FLAG ]] } +function fluentd_is_enabled() { + [[ -f $FLUENTD_FLAG ]] +} + rm -f $TELEGRAF_CONFIG if engine_is_object_based; then @@ -51,16 +57,33 @@ else fi fi -if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then +has_output=false + +# Include storage I/O inputs (estat_nfs, estat_iscsi, estat_backend-io) whenever +# any active output consumes those measurements — either InfluxDB or fluentd. +if { influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; } || + { fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; }; then if [[ -f $STORAGE_IO_INPUTS ]]; then cat $STORAGE_IO_INPUTS >> $TELEGRAF_CONFIG fi +fi + +if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then cat $INFLUXDB_OUTPUT >> $TELEGRAF_CONFIG + has_output=true else if influxdb_is_enabled; then logger -t delphix-telegraf "WARNING: INFLUXDB_ENABLED is set but $INFLUXDB_OUTPUT is missing — metrics will be discarded. Run delphix-influxdb-init to restore InfluxDB output." fi - # No InfluxDB output configured. Add discard so Telegraf can start — +fi + +if fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; then + cat $FLUENTD_OUTPUT >> $TELEGRAF_CONFIG + has_output=true +fi + +if [[ "$has_output" != "true" ]]; then + # No output configured. Add discard so Telegraf can start — # it requires at least one output plugin. echo "[[outputs.discard]]" >> $TELEGRAF_CONFIG fi diff --git a/telegraf/perf_fluentd b/telegraf/perf_fluentd new file mode 100644 index 0000000..7c9b8fc --- /dev/null +++ b/telegraf/perf_fluentd @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright (c) 2026 by Delphix. All rights reserved. +# +# Script that enables and disables fluentd 3rd-party metric forwarding for Telegraf. +# + +FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED + +function die() { + echo -e "$(date +%T:%N:%z): $(basename $0): $*" >&2 + exit 1 +} + +[[ $EUID -ne 0 ]] && die "must be run as root" + +function usage() { + echo "$(basename $0): $*" >&2 + echo "Usage: $(basename $0) [enable|disable]" + exit 2 +} + +function enable_fluentd() { + date + echo "Enabling fluentd 3rd-party metric forwarding" + touch $FLUENTD_FLAG + systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf" +} + +function disable_fluentd() { + date + echo "Disabling fluentd 3rd-party metric forwarding" + rm -f $FLUENTD_FLAG + systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf" +} + +if [[ $# -ne 1 ]]; then + usage +fi + +case "$1" in +enable) enable_fluentd ;; +disable) disable_fluentd ;; +*) usage ;; +esac diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd new file mode 100644 index 0000000..372b1f0 --- /dev/null +++ b/telegraf/telegraf.outputs.fluentd @@ -0,0 +1,13 @@ +# Telegraf HTTP output → local fluentd HTTP source. +# Forwards performance analytics metrics to fluentd, which routes them to +# third-party monitoring systems (Splunk, Datadog) via existing +# rules in fluent.conf.stg. +# Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists. +[[outputs.http]] + url = "http://localhost:9880/delphix.metrics.platform" + method = "POST" + data_format = "json" + flush_interval = "60s" + namepass = ["cpu", "diskio", "net", "zfs", "estat_nfs", "estat_iscsi"] + [outputs.http.headers] + Content-Type = "application/json" From d466d863c3b81eb7e3b170a3ce539d3bd11ad383 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 14:34:34 +0530 Subject: [PATCH 02/17] DLPX-100030 telegraf.outputs.fluentd: fix batch envelope format for fluentd Add json_transformation = "metrics" to strip the {"metrics":[...]} wrapper and send a top-level array that fluentd emits as individual metric records. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/telegraf.outputs.fluentd | 1 + 1 file changed, 1 insertion(+) diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd index 372b1f0..a5ec96a 100644 --- a/telegraf/telegraf.outputs.fluentd +++ b/telegraf/telegraf.outputs.fluentd @@ -7,6 +7,7 @@ url = "http://localhost:9880/delphix.metrics.platform" method = "POST" data_format = "json" + json_transformation = "metrics" flush_interval = "60s" namepass = ["cpu", "diskio", "net", "zfs", "estat_nfs", "estat_iscsi"] [outputs.http.headers] From d806670d4fdd3929fc234d018f404016c5835858 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 15:12:31 +0530 Subject: [PATCH 03/17] DLPX-100030 telegraf: splunkmetric format + engine identity global tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change 1: Switch fluentd output to data_format=splunkmetric with splunkmetric_hec_routing=true. Produces one HEC event per metric field (metric_name, _value, _time) — same structure Java's InsightLogger sent. Requires app-gate PR #5043 (data_type metric in fluentd match block). Change 2: Add telegraf.global_tags with systemUniqueId/systemVersion, included when fluentd is enabled. delphix-telegraf-service exports DELPHIX_SYSTEM_UUID (get-system-uuid) and DELPHIX_SYSTEM_VERSION (get-appliance-version) at startup so Telegraf can interpolate them. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/delphix-telegraf-service | 7 +++++++ telegraf/telegraf.global_tags | 6 ++++++ telegraf/telegraf.outputs.fluentd | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 telegraf/telegraf.global_tags diff --git a/telegraf/delphix-telegraf-service b/telegraf/delphix-telegraf-service index b3a07f0..382ec78 100755 --- a/telegraf/delphix-telegraf-service +++ b/telegraf/delphix-telegraf-service @@ -6,6 +6,7 @@ PLAYBOOK_INPUTS=/etc/telegraf/telegraf.inputs.playbook STORAGE_IO_INPUTS=/etc/telegraf/telegraf.inputs.storage_io INFLUXDB_OUTPUT=/etc/telegraf/telegraf.outputs.influxdb FLUENTD_OUTPUT=/etc/telegraf/telegraf.outputs.fluentd +GLOBAL_TAGS=/etc/telegraf/telegraf.global_tags PLAYBOOK_FLAG=/etc/telegraf/PLAYBOOK_ENABLED INFLUXDB_FLAG=/etc/telegraf/INFLUXDB_ENABLED FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED @@ -13,6 +14,9 @@ TELEGRAF_CONFIG=/etc/telegraf/telegraf.conf TELEGRAF_CONFIG_DIR=/etc/telegraf/telegraf.d +export DELPHIX_SYSTEM_UUID=$(get-system-uuid 2>/dev/null || echo "unknown") +export DELPHIX_SYSTEM_VERSION=$(get-appliance-version 2>/dev/null || echo "unknown") + function engine_is_dct() { /usr/bin/get-appliance-variant | grep -q "dct" >/dev/null [[ "$?" == "0" ]] @@ -78,6 +82,9 @@ else fi if fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; then + if [[ -f $GLOBAL_TAGS ]]; then + cat $GLOBAL_TAGS >> $TELEGRAF_CONFIG + fi cat $FLUENTD_OUTPUT >> $TELEGRAF_CONFIG has_output=true fi diff --git a/telegraf/telegraf.global_tags b/telegraf/telegraf.global_tags new file mode 100644 index 0000000..27eb84e --- /dev/null +++ b/telegraf/telegraf.global_tags @@ -0,0 +1,6 @@ +# Engine identity tags injected when fluentd 3rd-party output is enabled. +# DELPHIX_SYSTEM_UUID and DELPHIX_SYSTEM_VERSION are exported by +# delphix-telegraf-service at startup from get-system-uuid / get-appliance-version. +[global_tags] + systemUniqueId = "${DELPHIX_SYSTEM_UUID}" + systemVersion = "${DELPHIX_SYSTEM_VERSION}" diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd index a5ec96a..ad09fac 100644 --- a/telegraf/telegraf.outputs.fluentd +++ b/telegraf/telegraf.outputs.fluentd @@ -6,8 +6,8 @@ [[outputs.http]] url = "http://localhost:9880/delphix.metrics.platform" method = "POST" - data_format = "json" - json_transformation = "metrics" + data_format = "splunkmetric" + splunkmetric_hec_routing = true flush_interval = "60s" namepass = ["cpu", "diskio", "net", "zfs", "estat_nfs", "estat_iscsi"] [outputs.http.headers] From ae339a38f9b20d1e2dc6e8aaccfad5cb50b51422 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 20:51:46 +0530 Subject: [PATCH 04/17] DLPX-100030 telegraf: backward-compat Java metric names via starlark processor Add processors.starlark (telegraf.processors.fluentd) that maps Telegraf measurements to Java InsightLogger metric names for fluentd output: system.cpu.util.pct = 100 - cpu.usage_idle (cpu-total) system.net.total.bytes = sum(bytes_recv+bytes_sent) across interfaces system.disk.ops.count = sum(reads+writes) across devices system.nfs.ops.count = estat_nfs.read_ops + write_ops system.iscsi.ops.count = estat_iscsi.read_ops + write_ops system.tcp.total.bytes = sum(inbytes+outbytes) across tcp endpoints Cross-device aggregation uses Telegraf's built-in state dict with 10s timestamp rounding to handle sub-second timing differences across devices. Each metric type keeps its original measurement flowing to InfluxDB; only the Java-named metrics reach the fluentd output (namepass updated). Verified on dbs-test.dlpxdc.co: all 4 testable metrics confirmed correct (nfs/iscsi require active NFS/iSCSI activity, not present on test engine). Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/delphix-telegraf-service | 4 ++ telegraf/telegraf.outputs.fluentd | 3 +- telegraf/telegraf.processors.fluentd | 84 ++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 telegraf/telegraf.processors.fluentd diff --git a/telegraf/delphix-telegraf-service b/telegraf/delphix-telegraf-service index 382ec78..eecbc59 100755 --- a/telegraf/delphix-telegraf-service +++ b/telegraf/delphix-telegraf-service @@ -6,6 +6,7 @@ PLAYBOOK_INPUTS=/etc/telegraf/telegraf.inputs.playbook STORAGE_IO_INPUTS=/etc/telegraf/telegraf.inputs.storage_io INFLUXDB_OUTPUT=/etc/telegraf/telegraf.outputs.influxdb FLUENTD_OUTPUT=/etc/telegraf/telegraf.outputs.fluentd +FLUENTD_PROCESSOR=/etc/telegraf/telegraf.processors.fluentd GLOBAL_TAGS=/etc/telegraf/telegraf.global_tags PLAYBOOK_FLAG=/etc/telegraf/PLAYBOOK_ENABLED INFLUXDB_FLAG=/etc/telegraf/INFLUXDB_ENABLED @@ -85,6 +86,9 @@ if fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; then if [[ -f $GLOBAL_TAGS ]]; then cat $GLOBAL_TAGS >> $TELEGRAF_CONFIG fi + if [[ -f $FLUENTD_PROCESSOR ]]; then + cat $FLUENTD_PROCESSOR >> $TELEGRAF_CONFIG + fi cat $FLUENTD_OUTPUT >> $TELEGRAF_CONFIG has_output=true fi diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd index ad09fac..0118aa0 100644 --- a/telegraf/telegraf.outputs.fluentd +++ b/telegraf/telegraf.outputs.fluentd @@ -9,6 +9,7 @@ data_format = "splunkmetric" splunkmetric_hec_routing = true flush_interval = "60s" - namepass = ["cpu", "diskio", "net", "zfs", "estat_nfs", "estat_iscsi"] + namepass = ["system.cpu.util", "system.net.total", "system.disk.ops", + "system.nfs.ops", "system.iscsi.ops", "system.tcp.total"] [outputs.http.headers] Content-Type = "application/json" diff --git a/telegraf/telegraf.processors.fluentd b/telegraf/telegraf.processors.fluentd new file mode 100644 index 0000000..c40502e --- /dev/null +++ b/telegraf/telegraf.processors.fluentd @@ -0,0 +1,84 @@ +# Starlark processor — maps Telegraf measurements to Java InsightLogger metric names +# for backward-compatible fluentd output. +# +# splunkmetric_hec_routing forms metric_name as "measurement.field". To produce +# e.g. "system.cpu.util.pct", measurement = "system.cpu.util" and field = "pct". +# +# Net, disk, tcp require cross-device/interface aggregation. Telegraf's built-in +# `state` dict is used for mutable persistence across apply() calls (module-level +# dicts are frozen by Starlark and cannot be mutated). Each metric type stores +# its current-batch timestamp and running total in flat state keys, emitting the +# previous batch's total when a new timestamp is seen. +# +# Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists. +[[processors.starlark]] + namepass = ["cpu", "diskio", "net", "estat_nfs", "estat_iscsi", "tcp_stats"] + source = ''' +def accum(ts_key, val_key, out_measurement, out_field, ts, value): + # Round to 10s bucket so all devices/interfaces in one collection + # run share the same key regardless of sub-second timing differences. + bucket_ts = ts - (ts % 10) + result = [] + if ts_key not in state: + state[ts_key] = bucket_ts + state[val_key] = float(value) + elif bucket_ts == state[ts_key]: + state[val_key] = state[val_key] + float(value) + else: + m = Metric(out_measurement) + m.time = state[ts_key] + m.fields[out_field] = state[val_key] + result.append(m) + state[ts_key] = bucket_ts + state[val_key] = float(value) + return result + +def apply(metric): + name = metric.name + ts = metric.time + + # system.cpu.util.pct = 100 - usage_idle (cpu-total only) + if name == "cpu": + if metric.tags.get("cpu") == "cpu-total": + idle = float(metric.fields.get("usage_idle", 0)) + m = Metric("system.cpu.util") + m.time = ts + m.fields["pct"] = 100.0 - idle + return [metric, m] + return [metric] + + # system.nfs.ops.count = read_ops + write_ops + if name == "estat_nfs": + r = int(metric.fields.get("read_ops", 0)) + w = int(metric.fields.get("write_ops", 0)) + m = Metric("system.nfs.ops") + m.time = ts + m.fields["count"] = float(r + w) + return [metric, m] + + # system.iscsi.ops.count = read_ops + write_ops + if name == "estat_iscsi": + r = int(metric.fields.get("read_ops", 0)) + w = int(metric.fields.get("write_ops", 0)) + m = Metric("system.iscsi.ops") + m.time = ts + m.fields["count"] = float(r + w) + return [metric, m] + + # system.net.total.bytes = sum(bytes_recv + bytes_sent) across all interfaces + if name == "net": + val = int(metric.fields.get("bytes_recv", 0)) + int(metric.fields.get("bytes_sent", 0)) + return [metric] + accum("net_ts", "net_val", "system.net.total", "bytes", ts, val) + + # system.disk.ops.count = sum(reads + writes) across all devices + if name == "diskio": + val = int(metric.fields.get("reads", 0)) + int(metric.fields.get("writes", 0)) + return [metric] + accum("disk_ts", "disk_val", "system.disk.ops", "count", ts, val) + + # system.tcp.total.bytes = sum(inbytes + outbytes) across all TCP endpoints + if name == "tcp_stats": + val = int(metric.fields.get("inbytes", 0)) + int(metric.fields.get("outbytes", 0)) + return [metric] + accum("tcp_ts", "tcp_val", "system.tcp.total", "bytes", ts, val) + + return [metric] +''' From 1c08bb84a0657974788fb5ec64e6da6ae257201e Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:18:35 +0530 Subject: [PATCH 05/17] DLPX-100030 telegraf: fix delta semantics, bucket aggregation, review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconciled implementation with Java InsightLogger semantics (confirmed by reading dlpx-app-gate statcollector source): - net/diskio/tcp fields in Telegraf are cumulative kernel counters (same source as Java's /proc/net/dev etc.); Java computed deltas between consecutive polls. The starlark processor now does the same via `state`, keyed per interface / device / tcp endpoint. First-sample delta is 0; counter wrap guarded to 0. - cpu usage_idle is already a per-window percentage, keep as gauge. - estat_nfs/iscsi: fixed field name (iops(/s), not read_ops), multiplied by the 10s collection window to get ops-in-window. - All bucketing moved into starlark (no aggregator plugin) — basicstats would have suffixed field names with "_sum", corrupting the splunkmetric-derived metric_name. Uses nanosecond timestamps correctly (ts // 60e9). - systemUniqueId / systemVersion scoped to the compat metrics inside starlark via envsubst of DELPHIX_SYSTEM_UUID / DELPHIX_SYSTEM_VERSION — no longer Telegraf-global, so InfluxDB cardinality is unaffected (dropped telegraf.global_tags). - Added logger warnings for missing FLUENTD_OUTPUT / FLUENTD_PROCESSOR files (asymmetric fallback with InfluxDB path is now symmetric). - outputs.http: added timeout=10s and metric_buffer_limit=2048 so brief fluentd restarts don't lose data. - perf_fluentd: added status subcommand, idempotency check (no needless telegraf restart when flag is already in desired state), explicit die on touch failure. - Added explicit None guards for missing cpu/estat fields (previously returned 0, producing spurious 100% CPU or 0-ops metrics). Verified on dbs-test.dlpxdc.co: exactly one event per Java metric name per minute, with correct metric_name/_value/systemUniqueId/systemVersion fields matching Java InsightLogger format. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/delphix-telegraf-service | 15 ++- telegraf/perf_fluentd | 25 +++- telegraf/telegraf.global_tags | 6 - telegraf/telegraf.outputs.fluentd | 5 + telegraf/telegraf.processors.fluentd | 193 ++++++++++++++++++--------- 5 files changed, 169 insertions(+), 75 deletions(-) delete mode 100644 telegraf/telegraf.global_tags diff --git a/telegraf/delphix-telegraf-service b/telegraf/delphix-telegraf-service index eecbc59..32bc2fa 100755 --- a/telegraf/delphix-telegraf-service +++ b/telegraf/delphix-telegraf-service @@ -7,7 +7,6 @@ STORAGE_IO_INPUTS=/etc/telegraf/telegraf.inputs.storage_io INFLUXDB_OUTPUT=/etc/telegraf/telegraf.outputs.influxdb FLUENTD_OUTPUT=/etc/telegraf/telegraf.outputs.fluentd FLUENTD_PROCESSOR=/etc/telegraf/telegraf.processors.fluentd -GLOBAL_TAGS=/etc/telegraf/telegraf.global_tags PLAYBOOK_FLAG=/etc/telegraf/PLAYBOOK_ENABLED INFLUXDB_FLAG=/etc/telegraf/INFLUXDB_ENABLED FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED @@ -83,14 +82,18 @@ else fi if fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; then - if [[ -f $GLOBAL_TAGS ]]; then - cat $GLOBAL_TAGS >> $TELEGRAF_CONFIG - fi - if [[ -f $FLUENTD_PROCESSOR ]]; then - cat $FLUENTD_PROCESSOR >> $TELEGRAF_CONFIG + if [[ ! -f $FLUENTD_PROCESSOR ]]; then + logger -t delphix-telegraf "ERROR: FLUENTD_ENABLED is set but $FLUENTD_PROCESSOR is missing — raw Telegraf metric names will be forwarded, breaking existing Splunk dashboards. Reinstall the delphix-perfdiag package to restore it." + else + # Interpolate DELPHIX_SYSTEM_UUID / DELPHIX_SYSTEM_VERSION into the + # processor's starlark source. Telegraf's own ${VAR} substitution does + # not reach inside triple-quoted TOML strings, so we do it here. + envsubst '${DELPHIX_SYSTEM_UUID} ${DELPHIX_SYSTEM_VERSION}' < $FLUENTD_PROCESSOR >> $TELEGRAF_CONFIG fi cat $FLUENTD_OUTPUT >> $TELEGRAF_CONFIG has_output=true +elif fluentd_is_enabled; then + logger -t delphix-telegraf "WARNING: FLUENTD_ENABLED is set but $FLUENTD_OUTPUT is missing — metrics will be discarded. Reinstall the delphix-perfdiag package to restore fluentd output." fi if [[ "$has_output" != "true" ]]; then diff --git a/telegraf/perf_fluentd b/telegraf/perf_fluentd index 7c9b8fc..6ad851b 100644 --- a/telegraf/perf_fluentd +++ b/telegraf/perf_fluentd @@ -16,30 +16,49 @@ function die() { function usage() { echo "$(basename $0): $*" >&2 - echo "Usage: $(basename $0) [enable|disable]" + echo "Usage: $(basename $0) [enable|disable|status]" exit 2 } function enable_fluentd() { date + if [[ -f $FLUENTD_FLAG ]]; then + echo "fluentd forwarding already enabled — nothing to do" + return 0 + fi echo "Enabling fluentd 3rd-party metric forwarding" - touch $FLUENTD_FLAG + touch $FLUENTD_FLAG || die "Failed to create $FLUENTD_FLAG" systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf" } function disable_fluentd() { date + if [[ ! -f $FLUENTD_FLAG ]]; then + echo "fluentd forwarding already disabled — nothing to do" + return 0 + fi echo "Disabling fluentd 3rd-party metric forwarding" rm -f $FLUENTD_FLAG systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf" } +function status_fluentd() { + if [[ -f $FLUENTD_FLAG ]]; then + echo "fluentd forwarding: enabled" + else + echo "fluentd forwarding: disabled" + fi + echo -n "delphix-telegraf: " + systemctl is-active delphix-telegraf +} + if [[ $# -ne 1 ]]; then usage fi case "$1" in -enable) enable_fluentd ;; +enable) enable_fluentd ;; disable) disable_fluentd ;; +status) status_fluentd ;; *) usage ;; esac diff --git a/telegraf/telegraf.global_tags b/telegraf/telegraf.global_tags deleted file mode 100644 index 27eb84e..0000000 --- a/telegraf/telegraf.global_tags +++ /dev/null @@ -1,6 +0,0 @@ -# Engine identity tags injected when fluentd 3rd-party output is enabled. -# DELPHIX_SYSTEM_UUID and DELPHIX_SYSTEM_VERSION are exported by -# delphix-telegraf-service at startup from get-system-uuid / get-appliance-version. -[global_tags] - systemUniqueId = "${DELPHIX_SYSTEM_UUID}" - systemVersion = "${DELPHIX_SYSTEM_VERSION}" diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd index 0118aa0..ca9437c 100644 --- a/telegraf/telegraf.outputs.fluentd +++ b/telegraf/telegraf.outputs.fluentd @@ -9,6 +9,11 @@ data_format = "splunkmetric" splunkmetric_hec_routing = true flush_interval = "60s" + timeout = "10s" + # Buffer a few minutes of compat metrics so a brief fluentd restart doesn't + # lose data. The processor emits ~6 metrics per flush (one per Java metric + # name), so 2048 buffers > 5 hours of history — well above typical downtime. + metric_buffer_limit = 2048 namepass = ["system.cpu.util", "system.net.total", "system.disk.ops", "system.nfs.ops", "system.iscsi.ops", "system.tcp.total"] [outputs.http.headers] diff --git a/telegraf/telegraf.processors.fluentd b/telegraf/telegraf.processors.fluentd index c40502e..094541a 100644 --- a/telegraf/telegraf.processors.fluentd +++ b/telegraf/telegraf.processors.fluentd @@ -1,84 +1,157 @@ -# Starlark processor — maps Telegraf measurements to Java InsightLogger metric names -# for backward-compatible fluentd output. +# Starlark processor — maps Telegraf measurements to Java InsightLogger metric +# names (system.*) with semantics matching Java's pushInsightMetric: # -# splunkmetric_hec_routing forms metric_name as "measurement.field". To produce -# e.g. "system.cpu.util.pct", measurement = "system.cpu.util" and field = "pct". +# system.cpu.util.pct — % CPU busy in the window (gauge: last sample / 60s) +# system.net.total.bytes — DELTA bytes (recv+sent) over the window, all ifaces +# system.disk.ops.count — DELTA ops (read+write) over the window, all devices +# system.nfs.ops.count — ops (read+write) over the window across directions +# system.iscsi.ops.count — ops (read+write) over the window across directions +# system.tcp.total.bytes — DELTA bytes (in+out) over the window, all endpoints # -# Net, disk, tcp require cross-device/interface aggregation. Telegraf's built-in -# `state` dict is used for mutable persistence across apply() calls (module-level -# dicts are frozen by Starlark and cannot be mutated). Each metric type stores -# its current-batch timestamp and running total in flat state keys, emitting the -# previous batch's total when a new timestamp is seen. +# Java polled /proc/net/dev, /proc/diskstats, etc. at ~60s cadence and sent +# (current_cumulative - previous_cumulative). This processor does the same: +# - net / diskio / tcp_stats fields are cumulative kernel counters; +# we subtract the previous poll's value per-tag to recover the per-sample +# delta, then roll up per-sample deltas into a 60s bucket. +# - estat_nfs/_iscsi expose iops(/s), a rate averaged over the 10s collection +# window; ops-in-window = iops * 10, summed into the 60s bucket. +# - cpu usage_idle is already a percentage over the sample window — we keep +# the last sample's value in the 60s bucket (Java's gauge semantics). +# +# All bucketing happens here (no aggregator plugin) because Telegraf's +# basicstats aggregator appends "_sum" to field names, which would corrupt the +# splunkmetric-derived metric_name. Bucket emission is one 60s cycle behind +# (previous bucket emitted when the first sample of the next bucket arrives). +# +# systemUniqueId / systemVersion are scoped to the compat metrics only (set +# here, not as global_tags) so InfluxDB cardinality is unaffected. +# ${DELPHIX_SYSTEM_UUID} / ${DELPHIX_SYSTEM_VERSION} are substituted at TOML +# parse time from env vars exported by delphix-telegraf-service. # # Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists. [[processors.starlark]] namepass = ["cpu", "diskio", "net", "estat_nfs", "estat_iscsi", "tcp_stats"] source = ''' -def accum(ts_key, val_key, out_measurement, out_field, ts, value): - # Round to 10s bucket so all devices/interfaces in one collection - # run share the same key regardless of sub-second timing differences. - bucket_ts = ts - (ts % 10) - result = [] - if ts_key not in state: - state[ts_key] = bucket_ts - state[val_key] = float(value) - elif bucket_ts == state[ts_key]: - state[val_key] = state[val_key] + float(value) - else: - m = Metric(out_measurement) - m.time = state[ts_key] - m.fields[out_field] = state[val_key] - result.append(m) - state[ts_key] = bucket_ts - state[val_key] = float(value) - return result +UUID = "${DELPHIX_SYSTEM_UUID}" +VERSION = "${DELPHIX_SYSTEM_VERSION}" +BUCKET_NS = 60 * 1000 * 1000 * 1000 + +def tag_identity(m): + m.tags["systemUniqueId"] = UUID + m.tags["systemVersion"] = VERSION + return m + +def cumulative_delta(key, current): + # Delta since the previous call for this key, or 0 on the first call + # (no previous baseline). Guards against counter wrap / device reset. + prev = state.get(key, None) + state[key] = current + if prev == None: + return 0 + d = current - prev + if d < 0: + return 0 + return d + +def flush_counter(prefix, measurement, field, bucket): + # If we have crossed a 60s bucket boundary, emit the previous bucket's + # accumulated sum and reset the accumulator for the new bucket. Returns a + # list (possibly empty) of metrics to emit. + bk = prefix + "_bucket" + ak = prefix + "_acc" + prev_bucket = state.get(bk, None) + out = [] + if prev_bucket != None and prev_bucket != bucket: + acc = state.get(ak, 0.0) + m = Metric(measurement) + m.time = prev_bucket * BUCKET_NS + m.fields[field] = acc + out.append(tag_identity(m)) + state[ak] = 0.0 + state[bk] = bucket + return out + +def flush_gauge(prefix, measurement, field, bucket): + # Gauge version: emit the previous bucket's last-seen value when the + # bucket boundary is crossed. "last" is updated after this call. + bk = prefix + "_bucket" + lk = prefix + "_last" + prev_bucket = state.get(bk, None) + out = [] + if prev_bucket != None and prev_bucket != bucket and lk in state: + m = Metric(measurement) + m.time = prev_bucket * BUCKET_NS + m.fields[field] = state[lk] + out.append(tag_identity(m)) + state[bk] = bucket + return out def apply(metric): - name = metric.name - ts = metric.time + name = metric.name + ts = metric.time + bucket = ts // BUCKET_NS - # system.cpu.util.pct = 100 - usage_idle (cpu-total only) + # system.cpu.util.pct — gauge; keep last cpu-total sample per minute. if name == "cpu": - if metric.tags.get("cpu") == "cpu-total": - idle = float(metric.fields.get("usage_idle", 0)) - m = Metric("system.cpu.util") - m.time = ts - m.fields["pct"] = 100.0 - idle - return [metric, m] - return [metric] + if metric.tags.get("cpu") != "cpu-total": + return [metric] + if "usage_idle" not in metric.fields: + return [metric] + out = [metric] + out = out + flush_gauge("cpu", "system.cpu.util", "pct", bucket) + state["cpu_last"] = 100.0 - float(metric.fields["usage_idle"]) + return out - # system.nfs.ops.count = read_ops + write_ops + # system.nfs.ops.count — sum ops across read/write rows and 10s windows. if name == "estat_nfs": - r = int(metric.fields.get("read_ops", 0)) - w = int(metric.fields.get("write_ops", 0)) - m = Metric("system.nfs.ops") - m.time = ts - m.fields["count"] = float(r + w) - return [metric, m] + if "iops(/s)" not in metric.fields: + return [metric] + out = [metric] + out = out + flush_counter("nfs", "system.nfs.ops", "count", bucket) + state["nfs_acc"] = state.get("nfs_acc", 0.0) + float(metric.fields["iops(/s)"]) * 10.0 + return out - # system.iscsi.ops.count = read_ops + write_ops + # system.iscsi.ops.count — same shape as nfs. if name == "estat_iscsi": - r = int(metric.fields.get("read_ops", 0)) - w = int(metric.fields.get("write_ops", 0)) - m = Metric("system.iscsi.ops") - m.time = ts - m.fields["count"] = float(r + w) - return [metric, m] + if "iops(/s)" not in metric.fields: + return [metric] + out = [metric] + out = out + flush_counter("iscsi", "system.iscsi.ops", "count", bucket) + state["iscsi_acc"] = state.get("iscsi_acc", 0.0) + float(metric.fields["iops(/s)"]) * 10.0 + return out - # system.net.total.bytes = sum(bytes_recv + bytes_sent) across all interfaces + # system.net.total.bytes — per-interface cumulative-counter delta, summed. if name == "net": - val = int(metric.fields.get("bytes_recv", 0)) + int(metric.fields.get("bytes_sent", 0)) - return [metric] + accum("net_ts", "net_val", "system.net.total", "bytes", ts, val) + iface = metric.tags.get("interface", "") + recv = int(metric.fields.get("bytes_recv", 0)) + sent = int(metric.fields.get("bytes_sent", 0)) + d = cumulative_delta("net_r_" + iface, recv) + cumulative_delta("net_s_" + iface, sent) + out = [metric] + out = out + flush_counter("net", "system.net.total", "bytes", bucket) + state["net_acc"] = state.get("net_acc", 0.0) + float(d) + return out - # system.disk.ops.count = sum(reads + writes) across all devices + # system.disk.ops.count — per-device cumulative-counter delta, summed. if name == "diskio": - val = int(metric.fields.get("reads", 0)) + int(metric.fields.get("writes", 0)) - return [metric] + accum("disk_ts", "disk_val", "system.disk.ops", "count", ts, val) + dev = metric.tags.get("name", "") + r = int(metric.fields.get("reads", 0)) + w = int(metric.fields.get("writes", 0)) + d = cumulative_delta("disk_r_" + dev, r) + cumulative_delta("disk_w_" + dev, w) + out = [metric] + out = out + flush_counter("disk", "system.disk.ops", "count", bucket) + state["disk_acc"] = state.get("disk_acc", 0.0) + float(d) + return out - # system.tcp.total.bytes = sum(inbytes + outbytes) across all TCP endpoints + # system.tcp.total.bytes — per-endpoint cumulative-counter delta, summed. if name == "tcp_stats": - val = int(metric.fields.get("inbytes", 0)) + int(metric.fields.get("outbytes", 0)) - return [metric] + accum("tcp_ts", "tcp_val", "system.tcp.total", "bytes", ts, val) + key = metric.tags.get("laddr", "") + "|" + metric.tags.get("raddr", "") + "|" + metric.tags.get("service", "") + inb = int(metric.fields.get("inbytes", 0)) + outb = int(metric.fields.get("outbytes", 0)) + d = cumulative_delta("tcp_in_" + key, inb) + cumulative_delta("tcp_out_" + key, outb) + out = [metric] + out = out + flush_counter("tcp", "system.tcp.total", "bytes", bucket) + state["tcp_acc"] = state.get("tcp_acc", 0.0) + float(d) + return out return [metric] ''' From 7c30d2b9c4fd520d92a8983acaf1b83e2db4f48b Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:27:02 +0530 Subject: [PATCH 06/17] =?UTF-8?q?DLPX-100030=20docs:=20add=20HLD+LLD=20des?= =?UTF-8?q?ign=20document=20for=20telegraf=E2=86=92fluentd=20forwarding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers architecture, user controls, config assembly, metric transform pipeline, wire format comparison with Java InsightLogger, open questions, and a two-track stop-gap plan. Key context: DCT-managed engines already have zero fluentd forwarding (Java disabled on DCT add) — the Telegraf path is a bug fix for those customers, not a refactor. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 481 +++++++++++++++++++++++ 1 file changed, 481 insertions(+) create mode 100644 docs/fluentd-metric-forwarding-design.md diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md new file mode 100644 index 0000000..735a489 --- /dev/null +++ b/docs/fluentd-metric-forwarding-design.md @@ -0,0 +1,481 @@ +# Telegraf → Fluentd Metric Forwarding — Design Document + +**Status**: Draft — pending design review +**Jira**: [DLPX-100030](https://perforce.atlassian.net/browse/DLPX-100030) +**Author**: Dhruvin Shah +**Reviewers**: Seb Roy (architecture), TBD + +--- + +## 1. Executive Summary + +Delphix engines can forward performance metrics to customer-managed monitoring systems (Splunk, Datadog, Elasticsearch) via a Fluentd HTTP routing layer. This path was previously implemented in Java (`pushInsightMetric`). + +**Critical context:** When a Delphix Engine (DE) is added to DCT, Java performance collection is intentionally disabled on the engine to avoid data duplication (DCT becomes the collector). This means **customers who add their engine to DCT and rely on Splunk/Datadog forwarding immediately lose all metric data** — the Java path stops and there is currently no replacement. The Telegraf → Fluentd path is therefore not a refactor; it is a bug fix for an existing regression. + +This document covers the High-Level Design (HLD) and Low-Level Design (LLD) for the Telegraf-based replacement, the risks, the stop-gap position, and the acceptance criteria for a production release. + +--- + +## 2. Background — Why This Matters + +### 2.1 The Problem: DCT Breaks Third-Party Metric Forwarding + +When a DE is registered with DCT, the engine's Java analytics collection (`AnalyticsStorageImpl`, `LocalCpuCollector`, etc.) is disabled. This is intentional — DCT takes over and deduplicates the data. **However, `pushInsightMetric` — which forwards metrics to customer Splunk/Datadog — is also stopped as a side effect.** + +```mermaid +graph LR + subgraph Standalone["DE (standalone) — WORKS"] + J1["Java analytics\n(running)"] --> F1["FluentD"] --> M1["Splunk / Datadog\n✅ data flowing"] + end + + subgraph DCT["DE + DCT — BROKEN TODAY"] + J2["Java analytics\n(disabled — dedup)"] -. stopped .-> F2["FluentD"] --> M2["Splunk / Datadog\n❌ no data"] + T2["Telegraf\n(running — feeds DCT)"] -. no fluentd path .-> F2 + end +``` + +**Impact**: Any customer who has: +1. Enabled third-party metric forwarding (Splunk/Datadog), AND +2. Registered their engine with DCT + +is currently receiving zero metric data in their monitoring system. This is a silent regression — Telegraf runs fine but has no path to Fluentd. + +### 2.2 Why Replace It — The Real Driver Is DCT + +The primary motivation is **not** decommissioning Java for its own sake. It is: + +> When a DE is added to DCT, Java performance collection is intentionally stopped to prevent duplicate data. As a side effect, third-party metric forwarding (`pushInsightMetric`) also stops. Telegraf is already running on DCT-managed engines and is the only active collector — so Telegraf must take over the Fluentd forwarding role or those customers lose their Splunk/Datadog metrics entirely. + +Secondary benefits (valid, but not the driver): +- One collection stack instead of two (lower maintenance) +- Richer granularity in InfluxDB from Telegraf +- Pluggable output architecture + +### 2.3 The Current (Java) Path — Standalone Engines Only + +For engines NOT registered with DCT, the Java path still works. Customers receive one metric event per minute for six engine-level KPIs: + +``` +Java (AnalyticsStorageImpl) + │ + │ pushInsightMetric() — once per minute per metric + │ format: flat JSON over FluentD Forward Protocol (TCP 24224) + ▼ +FluentD (on engine) + │ + │ + │ routes to customer plugin + ▼ +Customer Monitoring System (Splunk, Datadog, etc.) +``` + +**Event shape Java sends:** + +```json +{ + "metric_name": "system.cpu.util.pct", + "_value": 80.4, + "_time": 1642520280000, + "event": "metric", + "type": "g", + "systemUniqueId": "564dec8e-f543-1aeb-7563-ee08901877c5", + "systemVersion": "6.0.1.0", + "data": { "user": 360, "kernel": 630, "idle": 118220 } +} +``` + +**The six Java metrics:** + +| metric_name | Meaning | Unit | +|---|---|---| +| `system.cpu.util.pct` | CPU busy % in last minute | gauge, % | +| `system.net.total.bytes` | Network bytes (in+out) in last minute | counter, bytes | +| `system.disk.ops.count` | Disk ops (read+write) in last minute | counter, ops | +| `system.nfs.ops.count` | NFS ops (read+write) in last minute | counter, ops | +| `system.iscsi.ops.count` | iSCSI ops (read+write) in last minute | counter, ops | +| `system.tcp.total.bytes` | TCP bytes (in+out) in last minute | counter, bytes | + +### 2.2 Why Replace It + +Java's collection stack (`AnalyticsStorageImpl`, `LocalCpuCollector`, etc.) is being decommissioned as Telegraf becomes the single source of truth for engine metrics. The Telegraf path offers: +- Lower maintenance cost (one collection stack, not two) +- Richer metric granularity in InfluxDB +- Pluggable output architecture + +--- + +## 3. High-Level Design (HLD) + +### 3.1 Architecture Overview + +```mermaid +graph TB + subgraph Engine["Delphix Engine"] + subgraph Inputs["Metric Collection (Telegraf inputs)"] + CPU["inputs.cpu\n10s interval"] + NET["inputs.net\n10s interval"] + DISK["inputs.diskio\n60s interval"] + NFS["inputs.execd\n(estat nfs)\n10s interval"] + ISCSI["inputs.execd\n(estat iscsi)\n10s interval"] + TCP["inputs.execd\n(connstat)\n10s interval"] + end + + subgraph Processing["Metric Transform (processors.starlark)"] + PROC["telegraf.processors.fluentd\n\n• Remap metric names to Java names\n• Compute deltas from cumulative counters\n• Roll up into 60s buckets\n• Inject engine identity tags\n\nOnly active when FLUENTD_ENABLED flag exists"] + end + + subgraph Outputs["Outputs"] + INFLUX["outputs.influxdb_v2\n(all raw metrics at full resolution)"] + HTTP["outputs.http\n→ localhost:9880\n(only system.* compat metrics,\none per minute)"] + end + + FLUENTD["FluentD\nHTTP source :9880\n\n"] + FLAG["/etc/telegraf/\nFLUENTD_ENABLED\n(flag file)"] + CLI["perf_fluentd\nenable | disable | status"] + end + + subgraph Customer["Customer Monitoring"] + SPLUNK["Splunk\n(mstats / SPL)"] + DATADOG["Datadog"] + ELASTIC["Elasticsearch"] + end + + CPU & NET & DISK & NFS & ISCSI & TCP --> PROC + CPU & NET & DISK & NFS & ISCSI & TCP --> INFLUX + PROC --> HTTP + HTTP --> FLUENTD + FLAG -.controls.-> PROC + CLI -.manages.-> FLAG + FLUENTD --> SPLUNK & DATADOG & ELASTIC +``` + +### 3.2 User Control Flow + +The customer or Delphix admin enables/disables forwarding via the setup screen. The app-gate calls `perf_fluentd enable/disable`. Telegraf restarts and re-reads the flag on every restart. + +```mermaid +sequenceDiagram + actor Admin as Admin / Setup Screen + participant AppGate as dlpx-app-gate + participant CLI as perf_fluentd + participant Flag as /etc/telegraf/FLUENTD_ENABLED + participant Service as delphix-telegraf-service + participant Telegraf as telegraf (process) + + Admin->>AppGate: Save third-party credentials + AppGate->>CLI: perf_fluentd enable + CLI->>Flag: touch (create flag file) + CLI->>Service: systemctl restart delphix-telegraf + Service->>Service: Read flag → include processor + output + Service->>Telegraf: Start with fluentd stanza in telegraf.conf + + Admin->>AppGate: Remove third-party credentials + AppGate->>CLI: perf_fluentd disable + CLI->>Flag: rm (remove flag file) + CLI->>Service: systemctl restart delphix-telegraf + Service->>Service: Flag absent → exclude processor + output + Service->>Telegraf: Start without fluentd stanza +``` + +### 3.3 telegraf.conf Assembly + +On every service start, `delphix-telegraf-service` generates `/etc/telegraf/telegraf.conf` by concatenating config fragments conditionally: + +```mermaid +flowchart TD + Start([Service starts]) --> Base[cat telegraf.base → telegraf.conf] + Base --> StorIO{InfluxDB OR\nFluentD enabled?} + StorIO -->|Yes| SI[append telegraf.inputs.storage_io\nest_nfs, estat_iscsi, tcp_stats] + StorIO -->|No| InfluxQ + + SI --> InfluxQ{INFLUXDB_ENABLED?} + InfluxQ -->|Yes + file exists| InfluxOut[append telegraf.outputs.influxdb] + InfluxQ -->|Yes + file missing| InfluxWarn[logger WARNING] + InfluxQ -->|No| FluentQ + + InfluxOut --> FluentQ + InfluxWarn --> FluentQ + + FluentQ{FLUENTD_ENABLED?} + FluentQ -->|Yes + output exists| ProcQ{FLUENTD_PROCESSOR\nexists?} + FluentQ -->|Yes + output missing| FluentWarn[logger WARNING] + FluentQ -->|No| HasOut + + ProcQ -->|Yes| Proc["envsubst UUID/VERSION\nappend telegraf.processors.fluentd"] + ProcQ -->|No| ProcErr[logger ERROR\nraw names will be forwarded] + + Proc --> FluentOut[append telegraf.outputs.fluentd] + ProcErr --> FluentOut + FluentOut --> HasOut + + HasOut{has_output\n= true?} + HasOut -->|No| Discard[echo outputs.discard] + HasOut -->|Yes| Perms + + Discard --> Perms[chmod 640 telegraf.conf] + Perms --> Launch[exec telegraf -config telegraf.conf] +``` + +--- + +## 4. Low-Level Design (LLD) + +### 4.1 Files Shipped by This Feature + +| File | Installed path | Purpose | +|---|---|---| +| `telegraf/perf_fluentd` | `/usr/bin/perf_fluentd` | Enable / disable / status CLI | +| `telegraf/telegraf.outputs.fluentd` | `/etc/telegraf/telegraf.outputs.fluentd` | HTTP output stanza (static) | +| `telegraf/telegraf.processors.fluentd` | `/etc/telegraf/telegraf.processors.fluentd` | Starlark metric transform | +| `telegraf/delphix-telegraf-service` | `/usr/bin/delphix-telegraf-service` | Modified: fluentd awareness | + +### 4.2 Metric Transform Pipeline (Starlark Processor) + +The starlark processor handles two classes of metrics differently: + +#### Class A — Gauge (instantaneous value) + +Only `system.cpu.util.pct`. Java computed `(user + kernel) / (user + kernel + idle) * 100` using delta jiffies over the sampling interval. Telegraf's `inputs.cpu` `usage_idle` field is already this ratio. We just invert it. + +``` +cpu (usage_idle = 5.3%) + └─ system.cpu.util.pct = 100 - 5.3 = 94.7% +``` + +Bucketing: keep the last sample in each 60s window and emit once at bucket transition. + +#### Class B — Counter (delta over the minute) + +Java polled `/proc/net/dev`, `/proc/diskstats`, etc. and sent `current - previous`. Telegraf exposes the same data as cumulative counters. We compute the delta in starlark using persistent state. + +``` +net (bytes_recv=1000, bytes_sent=500) ← cumulative since boot + prev bytes_recv=800, bytes_sent=350 ← stored in state + delta = (1000-800) + (500-350) = 350 bytes + └─ system.net.total.bytes += 350 ← add to 60s bucket +``` + +At bucket boundary: emit the accumulated bucket total, reset to 0. + +#### Class C — Rate-based counter (estat) + +`estat nfs/iscsi` output `iops(/s)` — the average rate over the 10s collection window. To get ops-in-window: `iops * 10`. Two rows per window (name=read, name=write), both accumulated. + +``` +estat_nfs (iops(/s)=50, name=read) → ops in 10s = 500 +estat_nfs (iops(/s)=30, name=write) → ops in 10s = 300 + accumulated into 60s bucket + └─ system.nfs.ops.count = 800 total ops this minute +``` + +#### Bucket state machine (all counter types) + +```mermaid +stateDiagram-v2 + [*] --> Waiting: Service starts + Waiting --> Accumulating: First sample arrives\n(no previous baseline, delta=0) + + Accumulating --> Accumulating: Sample in same 60s bucket\nadd delta to bucket_acc + + Accumulating --> Emitting: First sample of NEW 60s bucket arrives + Emitting --> Accumulating: Emit previous bucket total\nReset bucket_acc = 0\nAccumulate current sample + + note right of Waiting: state[*] = empty\nFirst delta always = 0\n(no prior baseline) + note right of Emitting: Emitted metric goes to\noutputs.http → fluentd +``` + +### 4.3 Event Format (LLD — what actually flows over the wire) + +Telegraf serialises using `splunkmetric_hec_routing = true`. For each field in each compat measurement, it emits one JSON object in Splunk HEC metrics format: + +**Wire format (HTTP POST body — newline-delimited HEC events):** + +```json +{"time":1793166060,"event":"metric","host":"ip-10-110-245-219","fields":{"metric_name":"system.cpu.util.pct","_value":94.7,"systemUniqueId":"564dec8e-...","systemVersion":"6.0.1.0"}} +{"time":1793166060,"event":"metric","host":"ip-10-110-245-219","fields":{"metric_name":"system.net.total.bytes","_value":350000,"systemUniqueId":"564dec8e-...","systemVersion":"6.0.1.0"}} +... +``` + +**Field-by-field comparison with Java:** + +| Field | Java | Telegraf (this design) | Compatible? | +|---|---|---|---| +| `metric_name` | `system.cpu.util.pct` | `system.cpu.util.pct` | ✅ exact match | +| `_value` | numeric | numeric (delta-based, same semantics) | ✅ | +| `_time` / `time` | milliseconds epoch | seconds epoch (HEC standard) | ⚠️ unit differs | +| `event` | `"metric"` | `"metric"` | ✅ exact match | +| `systemUniqueId` | engine UUID | engine UUID (via `get-system-uuid`) | ✅ exact match | +| `systemVersion` | `"6.0.1.0"` | full build string | ⚠️ format differs | +| `type` | `"g"` / `"c"` | absent | ⚠️ missing | +| `data` | `{"user":360, ...}` | absent | ❌ not replicated | +| Structure | flat JSON object | HEC nested `{"fields":{...}}` | ⚠️ needs `data_type metric` in fluent-plugin-splunk-hec | + +**Open items requiring design approval:** +1. `_time` unit — does fluent-plugin-splunk-hec accept seconds? Does Splunk mstats tolerate it? +2. `systemVersion` format — full build string vs. short version (`6.0.1.0`)? +3. `type` field — needed by any customer plugin? +4. `data` sub-object — any customer dashboard queries it? +5. Nested HEC structure — does the companion app-gate change (PR #5043) handle this? + +### 4.4 Identity Tag Injection + +`systemUniqueId` and `systemVersion` are set per compat metric inside the starlark source at config-generation time via `envsubst`. This scopes them to fluentd-bound metrics only (InfluxDB cardinality is unaffected). + +``` +delphix-telegraf-service + ├── reads env: DELPHIX_SYSTEM_UUID=$(get-system-uuid) + ├── reads env: DELPHIX_SYSTEM_VERSION=$(get-appliance-version) + └── envsubst '${DELPHIX_SYSTEM_UUID} ${DELPHIX_SYSTEM_VERSION}' \ + < telegraf.processors.fluentd >> telegraf.conf +``` + +### 4.5 Enable / Disable Control + +```mermaid +flowchart LR + subgraph perf_fluentd + E[enable] + D[disable] + S[status] + end + + E -->|idempotent:\nalready enabled?| NoOp1[print msg, exit 0] + E -->|not enabled| Touch["touch /etc/telegraf/FLUENTD_ENABLED\nrestart delphix-telegraf"] + + D -->|idempotent:\nalready disabled?| NoOp2[print msg, exit 0] + D -->|enabled| Remove["rm -f /etc/telegraf/FLUENTD_ENABLED\nrestart delphix-telegraf"] + + S --> Stat["print flag state\nprint systemctl is-active"] +``` + +--- + +## 5. Known Gaps and Open Questions + +These must be answered before this feature is promoted to GA: + +| # | Question | Owner | Impact | +|---|---|---|---| +| G1 | Does `_time` in seconds (vs Java's milliseconds) break Splunk mstats queries or Datadog ingest? | Seb | High — dashboard breakage | +| G2 | Does `systemVersion` as a full build string break customer dashboard filters? | Seb | Medium | +| G3 | Do any customer Splunk dashboards query the `data` sub-object (raw jiffy/bytes breakdown)? | Seb | Medium | +| G4 | Does `type: "g"/"c"` matter to any downstream plugin? | Seb | Low | +| G5 | Does the HEC nested structure (`fields: {...}`) work with existing customer `fluent-plugin-splunk-hec` configs without `data_type metric`? | Seb | High | +| G6 | Are `system.tcp.total.bytes` values from connstat semantically equivalent to what Java's `LocalTCPStatsCollector` sent? (cumulative per-connection vs delta per-endpoint) | Engineering | Medium | +| G7 | What is the validation plan for NFS/iSCSI metrics on an engine with active NFS/iSCSI traffic? | Engineering | High | + +--- + +## 6. Stop-Gap Position and Urgency + +### 6.1 Two Customer Segments, Different Urgency + +| Segment | Java path state | Impact today | Stop-gap available? | +|---|---|---|---| +| **DE standalone** (no DCT) | Running ✅ | No impact | Yes — Java continues | +| **DE + DCT** | Disabled ❌ | **Already broken** — zero metrics reaching Splunk/Datadog | **No stop-gap** — needs this fix | + +The "keep Java running in parallel" stop-gap only works for standalone engines. For DCT-managed engines, this PR **is** the stop-gap. Those customers are already getting zero data. + +### 6.2 Two-Track Approach + +```mermaid +graph TD + subgraph Track1["Track 1 — DCT engines (urgent)"] + A1["Answer open questions G1-G5\n(format compatibility)"] --> B1["Integration test on engine with active DCT"] + B1 --> C1["Enable perf_fluentd in app-gate\nfor DCT-managed engines only"] + C1 --> D1["Monitor customer Splunk/Datadog\nfor data continuity"] + end + + subgraph Track2["Track 2 — Standalone engines (measured)"] + A2["Full design sign-off"] --> B2["Customer beta test\n(volunteer Splunk customer)"] + B2 --> C2["Enable for standalone engines"] + end + + style Track1 fill:#ffdddd + style Track2 fill:#ddffdd +``` + +### 6.3 Recommended Stop-Gap Actions (Current Release) + +1. **Ship this code** — it is off by default and does not interfere with the Java path. +2. **DCT engines**: Prioritise answering G1 (time unit), G5 (HEC structure compatibility) — these are the only blockers for the DCT use case. Target: 1 sprint. +3. **Standalone engines**: Keep Java path active. Do NOT call `perf_fluentd enable` in app-gate for standalone engines until the full design is reviewed and tested. +4. **App-gate companion (PR #5043)**: Gated behind design review — do not merge until open questions are answered. + +```mermaid +gantt + title Recommended Timeline + dateFormat YYYY-MM-DD + + section Immediate (current release) + Ship code, fluentd off by default :done, 2026-10-28, 1d + Java continues for standalone engines :done, 2026-10-28, 60d + DCT engines: metrics broken (known gap) :crit, 2026-10-28, 14d + + section Sprint 1 (DCT fix — urgent) + Answer G1 + G5 (time unit, HEC compat) :active, 2026-11-01, 7d + Integration test: DCT + Splunk : 2026-11-08, 7d + Enable for DCT engines in app-gate : 2026-11-15, 1d + + section Sprint 2 (GA — standalone) + Full design review sign-off : 2026-11-17, 7d + Answer G2-G4, G6-G7 : 2026-11-17, 7d + Customer beta test (Splunk lab) : 2026-11-24, 14d + Enable for standalone engines : 2026-12-08, 1d +``` + +--- + +## 7. Testing Plan (for GA) + +| Test | What it verifies | +|---|---| +| `perf_fluentd enable` on a clean engine | Flag created, service restarts, fluentd stanza present in conf | +| `perf_fluentd disable` | Flag removed, stanza absent, InfluxDB output unaffected | +| Both InfluxDB + fluentd enabled | Both outputs in conf, storage_io inputs present, no discard | +| Neither enabled | `[[outputs.discard]]` in conf, Telegraf starts without error | +| Partial package (processor missing) | `logger` ERROR logged, service starts with raw names (documented fallback) | +| CPU metric value vs Java baseline | `system.cpu.util.pct` within ±1% of Java value at same point in time | +| Net/disk/tcp delta correctness | Download 100 MB, verify `system.net.total.bytes` ≈ 100 MB in that minute | +| NFS/iSCSI ops on active NFS mount | `system.nfs.ops.count` > 0 when NFS traffic active | +| Splunk end-to-end (lab) | Events appear in Splunk with correct `metric_name`, queryable via mstats | +| Datadog end-to-end (lab) | Events appear in Datadog metrics explorer | +| Service restart data gap | Restart Telegraf mid-minute; confirm at most one event missing | +| Concurrent enable/disable | Rapid enable/disable in loop; conf always valid, no half-written file | + +--- + +## 8. Risks + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| `_time` unit mismatch breaks Splunk mstats | Medium | High | Test in Splunk lab before GA; add explicit ms conversion if needed | +| `systemVersion` format breaks dashboard filters | Low | Medium | Strip to short form (e.g. `6.0.1.0`) at envsubst time | +| connstat instability causes `system.tcp.total.bytes` gaps | Medium | Low | Metric was absent in some Java releases too; acceptable | +| Starlark bucket logic has timing edge case at restart | Low | Low | At most one 60s gap; existing Java path also had restart gaps | +| InfluxDB receives `system.*` measurements (no namedrop) | High | Low | Extra series but no correctness impact; add namedrop in follow-up | +| NFS/iSCSI `iops(/s) * 10` breaks if estat interval changes | Low | Medium | Document coupling; add assertion in CI | + +--- + +## 9. Appendix — File Inventory + +``` +performance-diagnostics/ +├── telegraf/ +│ ├── perf_fluentd ← NEW: enable|disable|status CLI +│ ├── telegraf.outputs.fluentd ← NEW: HTTP output stanza +│ ├── telegraf.processors.fluentd ← NEW: starlark transform +│ └── delphix-telegraf-service ← MODIFIED: fluentd awareness +└── debian/ + └── rules ← MODIFIED: installs perf_fluentd +``` + +**Companion app-gate change (separate PR):** +``` +dlpx-app-gate/ +└── appliance/src/.../fluent.conf.stg ← adds + and data_type metric in +``` From 9b3aabf6db8066257fa91c209dedd03fa36027ba Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:29:32 +0530 Subject: [PATCH 07/17] DLPX-100030 docs: simplify HLD to three clean diagrams for laymen Replace the complex multi-subgraph architecture diagram, sequence diagram, and flowchart with three focused diagrams: 1. Single end-to-end flow (what it does in one picture) 2. On/off switch flow (how enable/disable works) 3. Before/after comparison (what stays the same for customers) Also corrected "Why Replace It" to lead with DCT context as the real driver. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 152 ++++++++++------------- 1 file changed, 64 insertions(+), 88 deletions(-) diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md index 735a489..6e778be 100644 --- a/docs/fluentd-metric-forwarding-design.md +++ b/docs/fluentd-metric-forwarding-design.md @@ -107,114 +107,90 @@ Java's collection stack (`AnalyticsStorageImpl`, `LocalCpuCollector`, etc.) is b ## 3. High-Level Design (HLD) -### 3.1 Architecture Overview +### 3.1 What It Does in One Picture ```mermaid -graph TB - subgraph Engine["Delphix Engine"] - subgraph Inputs["Metric Collection (Telegraf inputs)"] - CPU["inputs.cpu\n10s interval"] - NET["inputs.net\n10s interval"] - DISK["inputs.diskio\n60s interval"] - NFS["inputs.execd\n(estat nfs)\n10s interval"] - ISCSI["inputs.execd\n(estat iscsi)\n10s interval"] - TCP["inputs.execd\n(connstat)\n10s interval"] - end - - subgraph Processing["Metric Transform (processors.starlark)"] - PROC["telegraf.processors.fluentd\n\n• Remap metric names to Java names\n• Compute deltas from cumulative counters\n• Roll up into 60s buckets\n• Inject engine identity tags\n\nOnly active when FLUENTD_ENABLED flag exists"] - end - - subgraph Outputs["Outputs"] - INFLUX["outputs.influxdb_v2\n(all raw metrics at full resolution)"] - HTTP["outputs.http\n→ localhost:9880\n(only system.* compat metrics,\none per minute)"] - end - - FLUENTD["FluentD\nHTTP source :9880\n\n"] - FLAG["/etc/telegraf/\nFLUENTD_ENABLED\n(flag file)"] - CLI["perf_fluentd\nenable | disable | status"] - end +flowchart LR + Setup(["👤 Admin\nenables 3rd-party\nmonitoring\nin setup screen"]) - subgraph Customer["Customer Monitoring"] - SPLUNK["Splunk\n(mstats / SPL)"] - DATADOG["Datadog"] - ELASTIC["Elasticsearch"] + subgraph DE[" Delphix Engine "] + direction TB + Telegraf["📊 Telegraf\ncollects CPU, Network,\nDisk, NFS, iSCSI, TCP\nevery 10–60 seconds"] + Transform["🔄 Name Transform\nConverts to Java-compatible\nnames once per minute\neg. cpu → system.cpu.util.pct"] + Fluentd["📬 Fluentd\nroutes to customer\nmonitoring system"] end - CPU & NET & DISK & NFS & ISCSI & TCP --> PROC - CPU & NET & DISK & NFS & ISCSI & TCP --> INFLUX - PROC --> HTTP - HTTP --> FLUENTD - FLAG -.controls.-> PROC - CLI -.manages.-> FLAG - FLUENTD --> SPLUNK & DATADOG & ELASTIC -``` - -### 3.2 User Control Flow + Customer(["🖥️ Customer\nSplunk / Datadog\ndashboards"]) -The customer or Delphix admin enables/disables forwarding via the setup screen. The app-gate calls `perf_fluentd enable/disable`. Telegraf restarts and re-reads the flag on every restart. + Setup -->|"perf_fluentd enable\n(one-time setup)"| DE + Telegraf --> Transform + Transform -->|"HTTP POST\nlocalhost:9880\n1 event/min/metric"| Fluentd + Fluentd -->|"existing\ncustomer plugin"| Customer -```mermaid -sequenceDiagram - actor Admin as Admin / Setup Screen - participant AppGate as dlpx-app-gate - participant CLI as perf_fluentd - participant Flag as /etc/telegraf/FLUENTD_ENABLED - participant Service as delphix-telegraf-service - participant Telegraf as telegraf (process) - - Admin->>AppGate: Save third-party credentials - AppGate->>CLI: perf_fluentd enable - CLI->>Flag: touch (create flag file) - CLI->>Service: systemctl restart delphix-telegraf - Service->>Service: Read flag → include processor + output - Service->>Telegraf: Start with fluentd stanza in telegraf.conf - - Admin->>AppGate: Remove third-party credentials - AppGate->>CLI: perf_fluentd disable - CLI->>Flag: rm (remove flag file) - CLI->>Service: systemctl restart delphix-telegraf - Service->>Service: Flag absent → exclude processor + output - Service->>Telegraf: Start without fluentd stanza + style DE fill:#f0f4ff,stroke:#4466cc + style Transform fill:#fff8e0,stroke:#cc9900 ``` -### 3.3 telegraf.conf Assembly +**In plain English:** +1. Admin enables forwarding once in the Delphix setup screen. +2. Telegraf (already running) collects engine health metrics every few seconds. +3. A transform step (built into Telegraf) reshapes those metrics to match the exact format customers' Splunk/Datadog dashboards expect — one tidy event per metric per minute. +4. Fluentd (already running) receives the events and routes them to the customer's monitoring system using their existing plugin config. Nothing changes on the customer side. + +--- + +### 3.2 How the On/Off Switch Works -On every service start, `delphix-telegraf-service` generates `/etc/telegraf/telegraf.conf` by concatenating config fragments conditionally: +The customer can turn forwarding on or off from the Delphix setup screen at any time. ```mermaid flowchart TD - Start([Service starts]) --> Base[cat telegraf.base → telegraf.conf] - Base --> StorIO{InfluxDB OR\nFluentD enabled?} - StorIO -->|Yes| SI[append telegraf.inputs.storage_io\nest_nfs, estat_iscsi, tcp_stats] - StorIO -->|No| InfluxQ + A(["Admin saves\n3rd-party credentials\nin Setup UI"]) --> B["app-gate calls\nperf_fluentd enable"] + B --> C["Creates a flag file\n/etc/telegraf/FLUENTD_ENABLED"] + C --> D["Restarts Telegraf service"] + D --> E["Telegraf reads flag\nand activates forwarding"] + E --> F(["✅ Metrics flowing\nto Splunk / Datadog"]) + + G(["Admin removes\ncredentials"]) --> H["app-gate calls\nperf_fluentd disable"] + H --> I["Removes the flag file"] + I --> J["Restarts Telegraf service"] + J --> K(["⛔ Forwarding stopped\nInfluxDB unaffected"]) + + style F fill:#ddffdd,stroke:#44aa44 + style K fill:#ffdddd,stroke:#aa4444 +``` + +**Key point:** The flag file is the single source of truth. If it exists → forwarding is on. If it doesn't → forwarding is off. Telegraf reads this on every restart and builds its config accordingly. - SI --> InfluxQ{INFLUXDB_ENABLED?} - InfluxQ -->|Yes + file exists| InfluxOut[append telegraf.outputs.influxdb] - InfluxQ -->|Yes + file missing| InfluxWarn[logger WARNING] - InfluxQ -->|No| FluentQ +--- - InfluxOut --> FluentQ - InfluxWarn --> FluentQ +### 3.3 What Stays the Same for Customers - FluentQ{FLUENTD_ENABLED?} - FluentQ -->|Yes + output exists| ProcQ{FLUENTD_PROCESSOR\nexists?} - FluentQ -->|Yes + output missing| FluentWarn[logger WARNING] - FluentQ -->|No| HasOut +Customers do **not** change anything in their monitoring system. The event format, metric names, and Fluentd routing rules are preserved from the old Java implementation: - ProcQ -->|Yes| Proc["envsubst UUID/VERSION\nappend telegraf.processors.fluentd"] - ProcQ -->|No| ProcErr[logger ERROR\nraw names will be forwarded] +```mermaid +flowchart LR + subgraph Before["Before (Java path)"] + J["Java\npushInsightMetric()"] -->|"TCP 24224\nFluentd Forward"| F1["Fluentd"] --> S1["Splunk"] + end - Proc --> FluentOut[append telegraf.outputs.fluentd] - ProcErr --> FluentOut - FluentOut --> HasOut + subgraph After["After (Telegraf path) — this design"] + T["Telegraf\n+ transform"] -->|"HTTP 9880\nFluentd HTTP source"| F2["Fluentd"] --> S2["Splunk"] + end - HasOut{has_output\n= true?} - HasOut -->|No| Discard[echo outputs.discard] - HasOut -->|Yes| Perms + subgraph Same["What stays the same ✅"] + direction TB + N1["metric_name: system.cpu.util.pct"] + N2["_value: 94.7"] + N3["systemUniqueId: engine UUID"] + N4["event: metric"] + N5["1 event per metric per minute"] + N6["Fluentd match rules unchanged"] + end - Discard --> Perms[chmod 640 telegraf.conf] - Perms --> Launch[exec telegraf -config telegraf.conf] + style Before fill:#fff0f0 + style After fill:#f0fff0 + style Same fill:#f0f8ff ``` --- From 6f8b382aceeac0d5c0d3fef29f165b59b6cc63c9 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:30:51 +0530 Subject: [PATCH 08/17] DLPX-100030 docs: rewrite design doc as crisp 1-page decision brief MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 83 lines total. Problem → Solution → Data flow diagram → What customers receive → 3 new files → 5 open questions → recommendation table. Everything needed for a fast decision, nothing else. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 458 +++-------------------- 1 file changed, 42 insertions(+), 416 deletions(-) diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md index 6e778be..1b5ca04 100644 --- a/docs/fluentd-metric-forwarding-design.md +++ b/docs/fluentd-metric-forwarding-design.md @@ -1,457 +1,83 @@ -# Telegraf → Fluentd Metric Forwarding — Design Document +# Telegraf → Fluentd Metric Forwarding -**Status**: Draft — pending design review -**Jira**: [DLPX-100030](https://perforce.atlassian.net/browse/DLPX-100030) -**Author**: Dhruvin Shah -**Reviewers**: Seb Roy (architecture), TBD +**Jira**: [DLPX-100030](https://perforce.atlassian.net/browse/DLPX-100030) · **Status**: Pending review · **Owner**: Dhruvin Shah --- -## 1. Executive Summary +## Problem -Delphix engines can forward performance metrics to customer-managed monitoring systems (Splunk, Datadog, Elasticsearch) via a Fluentd HTTP routing layer. This path was previously implemented in Java (`pushInsightMetric`). +When a DE is added to DCT, Java analytics is disabled to avoid duplication. This silently stops metric forwarding to customer Splunk / Datadog — they receive nothing. -**Critical context:** When a Delphix Engine (DE) is added to DCT, Java performance collection is intentionally disabled on the engine to avoid data duplication (DCT becomes the collector). This means **customers who add their engine to DCT and rely on Splunk/Datadog forwarding immediately lose all metric data** — the Java path stops and there is currently no replacement. The Telegraf → Fluentd path is therefore not a refactor; it is a bug fix for an existing regression. - -This document covers the High-Level Design (HLD) and Low-Level Design (LLD) for the Telegraf-based replacement, the risks, the stop-gap position, and the acceptance criteria for a production release. - ---- - -## 2. Background — Why This Matters - -### 2.1 The Problem: DCT Breaks Third-Party Metric Forwarding - -When a DE is registered with DCT, the engine's Java analytics collection (`AnalyticsStorageImpl`, `LocalCpuCollector`, etc.) is disabled. This is intentional — DCT takes over and deduplicates the data. **However, `pushInsightMetric` — which forwards metrics to customer Splunk/Datadog — is also stopped as a side effect.** - -```mermaid -graph LR - subgraph Standalone["DE (standalone) — WORKS"] - J1["Java analytics\n(running)"] --> F1["FluentD"] --> M1["Splunk / Datadog\n✅ data flowing"] - end - - subgraph DCT["DE + DCT — BROKEN TODAY"] - J2["Java analytics\n(disabled — dedup)"] -. stopped .-> F2["FluentD"] --> M2["Splunk / Datadog\n❌ no data"] - T2["Telegraf\n(running — feeds DCT)"] -. no fluentd path .-> F2 - end ``` - -**Impact**: Any customer who has: -1. Enabled third-party metric forwarding (Splunk/Datadog), AND -2. Registered their engine with DCT - -is currently receiving zero metric data in their monitoring system. This is a silent regression — Telegraf runs fine but has no path to Fluentd. - -### 2.2 Why Replace It — The Real Driver Is DCT - -The primary motivation is **not** decommissioning Java for its own sake. It is: - -> When a DE is added to DCT, Java performance collection is intentionally stopped to prevent duplicate data. As a side effect, third-party metric forwarding (`pushInsightMetric`) also stops. Telegraf is already running on DCT-managed engines and is the only active collector — so Telegraf must take over the Fluentd forwarding role or those customers lose their Splunk/Datadog metrics entirely. - -Secondary benefits (valid, but not the driver): -- One collection stack instead of two (lower maintenance) -- Richer granularity in InfluxDB from Telegraf -- Pluggable output architecture - -### 2.3 The Current (Java) Path — Standalone Engines Only - -For engines NOT registered with DCT, the Java path still works. Customers receive one metric event per minute for six engine-level KPIs: - -``` -Java (AnalyticsStorageImpl) - │ - │ pushInsightMetric() — once per minute per metric - │ format: flat JSON over FluentD Forward Protocol (TCP 24224) - ▼ -FluentD (on engine) - │ - │ - │ routes to customer plugin - ▼ -Customer Monitoring System (Splunk, Datadog, etc.) -``` - -**Event shape Java sends:** - -```json -{ - "metric_name": "system.cpu.util.pct", - "_value": 80.4, - "_time": 1642520280000, - "event": "metric", - "type": "g", - "systemUniqueId": "564dec8e-f543-1aeb-7563-ee08901877c5", - "systemVersion": "6.0.1.0", - "data": { "user": 360, "kernel": 630, "idle": 118220 } -} -``` - -**The six Java metrics:** - -| metric_name | Meaning | Unit | -|---|---|---| -| `system.cpu.util.pct` | CPU busy % in last minute | gauge, % | -| `system.net.total.bytes` | Network bytes (in+out) in last minute | counter, bytes | -| `system.disk.ops.count` | Disk ops (read+write) in last minute | counter, ops | -| `system.nfs.ops.count` | NFS ops (read+write) in last minute | counter, ops | -| `system.iscsi.ops.count` | iSCSI ops (read+write) in last minute | counter, ops | -| `system.tcp.total.bytes` | TCP bytes (in+out) in last minute | counter, bytes | - -### 2.2 Why Replace It - -Java's collection stack (`AnalyticsStorageImpl`, `LocalCpuCollector`, etc.) is being decommissioned as Telegraf becomes the single source of truth for engine metrics. The Telegraf path offers: -- Lower maintenance cost (one collection stack, not two) -- Richer metric granularity in InfluxDB -- Pluggable output architecture - ---- - -## 3. High-Level Design (HLD) - -### 3.1 What It Does in One Picture - -```mermaid -flowchart LR - Setup(["👤 Admin\nenables 3rd-party\nmonitoring\nin setup screen"]) - - subgraph DE[" Delphix Engine "] - direction TB - Telegraf["📊 Telegraf\ncollects CPU, Network,\nDisk, NFS, iSCSI, TCP\nevery 10–60 seconds"] - Transform["🔄 Name Transform\nConverts to Java-compatible\nnames once per minute\neg. cpu → system.cpu.util.pct"] - Fluentd["📬 Fluentd\nroutes to customer\nmonitoring system"] - end - - Customer(["🖥️ Customer\nSplunk / Datadog\ndashboards"]) - - Setup -->|"perf_fluentd enable\n(one-time setup)"| DE - Telegraf --> Transform - Transform -->|"HTTP POST\nlocalhost:9880\n1 event/min/metric"| Fluentd - Fluentd -->|"existing\ncustomer plugin"| Customer - - style DE fill:#f0f4ff,stroke:#4466cc - style Transform fill:#fff8e0,stroke:#cc9900 +DE + DCT today: Java OFF → Fluentd → ❌ Splunk (no data) + Telegraf ON → InfluxDB ✅ (but no Fluentd path) ``` -**In plain English:** -1. Admin enables forwarding once in the Delphix setup screen. -2. Telegraf (already running) collects engine health metrics every few seconds. -3. A transform step (built into Telegraf) reshapes those metrics to match the exact format customers' Splunk/Datadog dashboards expect — one tidy event per metric per minute. -4. Fluentd (already running) receives the events and routes them to the customer's monitoring system using their existing plugin config. Nothing changes on the customer side. - --- -### 3.2 How the On/Off Switch Works - -The customer can turn forwarding on or off from the Delphix setup screen at any time. +## Solution -```mermaid -flowchart TD - A(["Admin saves\n3rd-party credentials\nin Setup UI"]) --> B["app-gate calls\nperf_fluentd enable"] - B --> C["Creates a flag file\n/etc/telegraf/FLUENTD_ENABLED"] - C --> D["Restarts Telegraf service"] - D --> E["Telegraf reads flag\nand activates forwarding"] - E --> F(["✅ Metrics flowing\nto Splunk / Datadog"]) - - G(["Admin removes\ncredentials"]) --> H["app-gate calls\nperf_fluentd disable"] - H --> I["Removes the flag file"] - I --> J["Restarts Telegraf service"] - J --> K(["⛔ Forwarding stopped\nInfluxDB unaffected"]) - - style F fill:#ddffdd,stroke:#44aa44 - style K fill:#ffdddd,stroke:#aa4444 -``` - -**Key point:** The flag file is the single source of truth. If it exists → forwarding is on. If it doesn't → forwarding is off. Telegraf reads this on every restart and builds its config accordingly. +Add a Fluentd output path to Telegraf so DCT-managed engines resume forwarding metrics to customer monitoring systems with **no changes on the customer side**. --- -### 3.3 What Stays the Same for Customers - -Customers do **not** change anything in their monitoring system. The event format, metric names, and Fluentd routing rules are preserved from the old Java implementation: +## Data Flow ```mermaid flowchart LR - subgraph Before["Before (Java path)"] - J["Java\npushInsightMetric()"] -->|"TCP 24224\nFluentd Forward"| F1["Fluentd"] --> S1["Splunk"] - end - - subgraph After["After (Telegraf path) — this design"] - T["Telegraf\n+ transform"] -->|"HTTP 9880\nFluentd HTTP source"| F2["Fluentd"] --> S2["Splunk"] - end + Admin(["Admin\nSetup Screen"]) -->|"enable/disable"| Toggle["FLUENTD_ENABLED\nflag file"] + Toggle -->|controls| T - subgraph Same["What stays the same ✅"] - direction TB - N1["metric_name: system.cpu.util.pct"] - N2["_value: 94.7"] - N3["systemUniqueId: engine UUID"] - N4["event: metric"] - N5["1 event per metric per minute"] - N6["Fluentd match rules unchanged"] + subgraph DE["Delphix Engine"] + T["Telegraf\ncollects metrics"] --> P["Transform\nrenames + rolls up\nto 1 event/min"] --> F["Fluentd\n:9880"] end - style Before fill:#fff0f0 - style After fill:#f0fff0 - style Same fill:#f0f8ff + F --> Customer(["Splunk / Datadog\n← unchanged"]) ``` --- -## 4. Low-Level Design (LLD) - -### 4.1 Files Shipped by This Feature - -| File | Installed path | Purpose | -|---|---|---| -| `telegraf/perf_fluentd` | `/usr/bin/perf_fluentd` | Enable / disable / status CLI | -| `telegraf/telegraf.outputs.fluentd` | `/etc/telegraf/telegraf.outputs.fluentd` | HTTP output stanza (static) | -| `telegraf/telegraf.processors.fluentd` | `/etc/telegraf/telegraf.processors.fluentd` | Starlark metric transform | -| `telegraf/delphix-telegraf-service` | `/usr/bin/delphix-telegraf-service` | Modified: fluentd awareness | - -### 4.2 Metric Transform Pipeline (Starlark Processor) - -The starlark processor handles two classes of metrics differently: - -#### Class A — Gauge (instantaneous value) - -Only `system.cpu.util.pct`. Java computed `(user + kernel) / (user + kernel + idle) * 100` using delta jiffies over the sampling interval. Telegraf's `inputs.cpu` `usage_idle` field is already this ratio. We just invert it. - -``` -cpu (usage_idle = 5.3%) - └─ system.cpu.util.pct = 100 - 5.3 = 94.7% -``` - -Bucketing: keep the last sample in each 60s window and emit once at bucket transition. - -#### Class B — Counter (delta over the minute) - -Java polled `/proc/net/dev`, `/proc/diskstats`, etc. and sent `current - previous`. Telegraf exposes the same data as cumulative counters. We compute the delta in starlark using persistent state. - -``` -net (bytes_recv=1000, bytes_sent=500) ← cumulative since boot - prev bytes_recv=800, bytes_sent=350 ← stored in state - delta = (1000-800) + (500-350) = 350 bytes - └─ system.net.total.bytes += 350 ← add to 60s bucket -``` - -At bucket boundary: emit the accumulated bucket total, reset to 0. +## What Customers Receive (unchanged from Java) -#### Class C — Rate-based counter (estat) - -`estat nfs/iscsi` output `iops(/s)` — the average rate over the 10s collection window. To get ops-in-window: `iops * 10`. Two rows per window (name=read, name=write), both accumulated. - -``` -estat_nfs (iops(/s)=50, name=read) → ops in 10s = 500 -estat_nfs (iops(/s)=30, name=write) → ops in 10s = 300 - accumulated into 60s bucket - └─ system.nfs.ops.count = 800 total ops this minute -``` - -#### Bucket state machine (all counter types) - -```mermaid -stateDiagram-v2 - [*] --> Waiting: Service starts - Waiting --> Accumulating: First sample arrives\n(no previous baseline, delta=0) - - Accumulating --> Accumulating: Sample in same 60s bucket\nadd delta to bucket_acc - - Accumulating --> Emitting: First sample of NEW 60s bucket arrives - Emitting --> Accumulating: Emit previous bucket total\nReset bucket_acc = 0\nAccumulate current sample - - note right of Waiting: state[*] = empty\nFirst delta always = 0\n(no prior baseline) - note right of Emitting: Emitted metric goes to\noutputs.http → fluentd -``` - -### 4.3 Event Format (LLD — what actually flows over the wire) - -Telegraf serialises using `splunkmetric_hec_routing = true`. For each field in each compat measurement, it emits one JSON object in Splunk HEC metrics format: - -**Wire format (HTTP POST body — newline-delimited HEC events):** - -```json -{"time":1793166060,"event":"metric","host":"ip-10-110-245-219","fields":{"metric_name":"system.cpu.util.pct","_value":94.7,"systemUniqueId":"564dec8e-...","systemVersion":"6.0.1.0"}} -{"time":1793166060,"event":"metric","host":"ip-10-110-245-219","fields":{"metric_name":"system.net.total.bytes","_value":350000,"systemUniqueId":"564dec8e-...","systemVersion":"6.0.1.0"}} -... -``` - -**Field-by-field comparison with Java:** - -| Field | Java | Telegraf (this design) | Compatible? | -|---|---|---|---| -| `metric_name` | `system.cpu.util.pct` | `system.cpu.util.pct` | ✅ exact match | -| `_value` | numeric | numeric (delta-based, same semantics) | ✅ | -| `_time` / `time` | milliseconds epoch | seconds epoch (HEC standard) | ⚠️ unit differs | -| `event` | `"metric"` | `"metric"` | ✅ exact match | -| `systemUniqueId` | engine UUID | engine UUID (via `get-system-uuid`) | ✅ exact match | -| `systemVersion` | `"6.0.1.0"` | full build string | ⚠️ format differs | -| `type` | `"g"` / `"c"` | absent | ⚠️ missing | -| `data` | `{"user":360, ...}` | absent | ❌ not replicated | -| Structure | flat JSON object | HEC nested `{"fields":{...}}` | ⚠️ needs `data_type metric` in fluent-plugin-splunk-hec | - -**Open items requiring design approval:** -1. `_time` unit — does fluent-plugin-splunk-hec accept seconds? Does Splunk mstats tolerate it? -2. `systemVersion` format — full build string vs. short version (`6.0.1.0`)? -3. `type` field — needed by any customer plugin? -4. `data` sub-object — any customer dashboard queries it? -5. Nested HEC structure — does the companion app-gate change (PR #5043) handle this? - -### 4.4 Identity Tag Injection - -`systemUniqueId` and `systemVersion` are set per compat metric inside the starlark source at config-generation time via `envsubst`. This scopes them to fluentd-bound metrics only (InfluxDB cardinality is unaffected). - -``` -delphix-telegraf-service - ├── reads env: DELPHIX_SYSTEM_UUID=$(get-system-uuid) - ├── reads env: DELPHIX_SYSTEM_VERSION=$(get-appliance-version) - └── envsubst '${DELPHIX_SYSTEM_UUID} ${DELPHIX_SYSTEM_VERSION}' \ - < telegraf.processors.fluentd >> telegraf.conf -``` - -### 4.5 Enable / Disable Control - -```mermaid -flowchart LR - subgraph perf_fluentd - E[enable] - D[disable] - S[status] - end - - E -->|idempotent:\nalready enabled?| NoOp1[print msg, exit 0] - E -->|not enabled| Touch["touch /etc/telegraf/FLUENTD_ENABLED\nrestart delphix-telegraf"] - - D -->|idempotent:\nalready disabled?| NoOp2[print msg, exit 0] - D -->|enabled| Remove["rm -f /etc/telegraf/FLUENTD_ENABLED\nrestart delphix-telegraf"] - - S --> Stat["print flag state\nprint systemctl is-active"] -``` - ---- - -## 5. Known Gaps and Open Questions - -These must be answered before this feature is promoted to GA: - -| # | Question | Owner | Impact | -|---|---|---|---| -| G1 | Does `_time` in seconds (vs Java's milliseconds) break Splunk mstats queries or Datadog ingest? | Seb | High — dashboard breakage | -| G2 | Does `systemVersion` as a full build string break customer dashboard filters? | Seb | Medium | -| G3 | Do any customer Splunk dashboards query the `data` sub-object (raw jiffy/bytes breakdown)? | Seb | Medium | -| G4 | Does `type: "g"/"c"` matter to any downstream plugin? | Seb | Low | -| G5 | Does the HEC nested structure (`fields: {...}`) work with existing customer `fluent-plugin-splunk-hec` configs without `data_type metric`? | Seb | High | -| G6 | Are `system.tcp.total.bytes` values from connstat semantically equivalent to what Java's `LocalTCPStatsCollector` sent? (cumulative per-connection vs delta per-endpoint) | Engineering | Medium | -| G7 | What is the validation plan for NFS/iSCSI metrics on an engine with active NFS/iSCSI traffic? | Engineering | High | - ---- - -## 6. Stop-Gap Position and Urgency - -### 6.1 Two Customer Segments, Different Urgency - -| Segment | Java path state | Impact today | Stop-gap available? | -|---|---|---|---| -| **DE standalone** (no DCT) | Running ✅ | No impact | Yes — Java continues | -| **DE + DCT** | Disabled ❌ | **Already broken** — zero metrics reaching Splunk/Datadog | **No stop-gap** — needs this fix | - -The "keep Java running in parallel" stop-gap only works for standalone engines. For DCT-managed engines, this PR **is** the stop-gap. Those customers are already getting zero data. - -### 6.2 Two-Track Approach - -```mermaid -graph TD - subgraph Track1["Track 1 — DCT engines (urgent)"] - A1["Answer open questions G1-G5\n(format compatibility)"] --> B1["Integration test on engine with active DCT"] - B1 --> C1["Enable perf_fluentd in app-gate\nfor DCT-managed engines only"] - C1 --> D1["Monitor customer Splunk/Datadog\nfor data continuity"] - end - - subgraph Track2["Track 2 — Standalone engines (measured)"] - A2["Full design sign-off"] --> B2["Customer beta test\n(volunteer Splunk customer)"] - B2 --> C2["Enable for standalone engines"] - end - - style Track1 fill:#ffdddd - style Track2 fill:#ddffdd -``` - -### 6.3 Recommended Stop-Gap Actions (Current Release) - -1. **Ship this code** — it is off by default and does not interfere with the Java path. -2. **DCT engines**: Prioritise answering G1 (time unit), G5 (HEC structure compatibility) — these are the only blockers for the DCT use case. Target: 1 sprint. -3. **Standalone engines**: Keep Java path active. Do NOT call `perf_fluentd enable` in app-gate for standalone engines until the full design is reviewed and tested. -4. **App-gate companion (PR #5043)**: Gated behind design review — do not merge until open questions are answered. - -```mermaid -gantt - title Recommended Timeline - dateFormat YYYY-MM-DD - - section Immediate (current release) - Ship code, fluentd off by default :done, 2026-10-28, 1d - Java continues for standalone engines :done, 2026-10-28, 60d - DCT engines: metrics broken (known gap) :crit, 2026-10-28, 14d - - section Sprint 1 (DCT fix — urgent) - Answer G1 + G5 (time unit, HEC compat) :active, 2026-11-01, 7d - Integration test: DCT + Splunk : 2026-11-08, 7d - Enable for DCT engines in app-gate : 2026-11-15, 1d - - section Sprint 2 (GA — standalone) - Full design review sign-off : 2026-11-17, 7d - Answer G2-G4, G6-G7 : 2026-11-17, 7d - Customer beta test (Splunk lab) : 2026-11-24, 14d - Enable for standalone engines : 2026-12-08, 1d -``` +| Field | Value | +|---|---| +| `metric_name` | `system.cpu.util.pct`, `system.net.total.bytes`, `system.disk.ops.count`, `system.nfs.ops.count`, `system.iscsi.ops.count`, `system.tcp.total.bytes` | +| `_value` | Numeric value for that metric | +| `systemUniqueId` | Engine UUID | +| `systemVersion` | Engine version | +| `event` | `"metric"` | +| Frequency | 1 event per metric per minute | +| Protocol | HTTP POST → Fluentd :9880 (was TCP :24224) | --- -## 7. Testing Plan (for GA) +## What's New (3 files) -| Test | What it verifies | +| File | What it does | |---|---| -| `perf_fluentd enable` on a clean engine | Flag created, service restarts, fluentd stanza present in conf | -| `perf_fluentd disable` | Flag removed, stanza absent, InfluxDB output unaffected | -| Both InfluxDB + fluentd enabled | Both outputs in conf, storage_io inputs present, no discard | -| Neither enabled | `[[outputs.discard]]` in conf, Telegraf starts without error | -| Partial package (processor missing) | `logger` ERROR logged, service starts with raw names (documented fallback) | -| CPU metric value vs Java baseline | `system.cpu.util.pct` within ±1% of Java value at same point in time | -| Net/disk/tcp delta correctness | Download 100 MB, verify `system.net.total.bytes` ≈ 100 MB in that minute | -| NFS/iSCSI ops on active NFS mount | `system.nfs.ops.count` > 0 when NFS traffic active | -| Splunk end-to-end (lab) | Events appear in Splunk with correct `metric_name`, queryable via mstats | -| Datadog end-to-end (lab) | Events appear in Datadog metrics explorer | -| Service restart data gap | Restart Telegraf mid-minute; confirm at most one event missing | -| Concurrent enable/disable | Rapid enable/disable in loop; conf always valid, no half-written file | +| `perf_fluentd` | CLI: `enable` / `disable` / `status` | +| `telegraf.outputs.fluentd` | HTTP output config (static) | +| `telegraf.processors.fluentd` | Starlark: rename metrics + compute 60s deltas | --- -## 8. Risks +## Open Questions — Need Answers Before GA -| Risk | Likelihood | Impact | Mitigation | +| # | Question | Who | Risk | |---|---|---|---| -| `_time` unit mismatch breaks Splunk mstats | Medium | High | Test in Splunk lab before GA; add explicit ms conversion if needed | -| `systemVersion` format breaks dashboard filters | Low | Medium | Strip to short form (e.g. `6.0.1.0`) at envsubst time | -| connstat instability causes `system.tcp.total.bytes` gaps | Medium | Low | Metric was absent in some Java releases too; acceptable | -| Starlark bucket logic has timing edge case at restart | Low | Low | At most one 60s gap; existing Java path also had restart gaps | -| InfluxDB receives `system.*` measurements (no namedrop) | High | Low | Extra series but no correctness impact; add namedrop in follow-up | -| NFS/iSCSI `iops(/s) * 10` breaks if estat interval changes | Low | Medium | Document coupling; add assertion in CI | +| Q1 | Does `_time` in **seconds** (not ms like Java) break Splunk mstats? | Seb | High | +| Q2 | Does the HEC nested structure `{"fields":{...}}` work with existing customer Fluentd plugins? | Seb | High | +| Q3 | Does `systemVersion` as a full build string break any dashboard filter? | Seb | Medium | +| Q4 | Is `system.tcp.total.bytes` from connstat semantically equivalent to Java's value? | Eng | Medium | +| Q5 | Can we test NFS/iSCSI metrics on an engine with active traffic before GA? | Eng | Medium | --- -## 9. Appendix — File Inventory +## Recommendation -``` -performance-diagnostics/ -├── telegraf/ -│ ├── perf_fluentd ← NEW: enable|disable|status CLI -│ ├── telegraf.outputs.fluentd ← NEW: HTTP output stanza -│ ├── telegraf.processors.fluentd ← NEW: starlark transform -│ └── delphix-telegraf-service ← MODIFIED: fluentd awareness -└── debian/ - └── rules ← MODIFIED: installs perf_fluentd -``` +| Segment | Today | Action | +|---|---|---| +| **DE + DCT** | ❌ Broken — no data in Splunk | Unblock: answer Q1 + Q2 this sprint, enable | +| **DE standalone** | ✅ Java still working | Hold: keep Java, enable Telegraf path only after Q1–Q5 answered | -**Companion app-gate change (separate PR):** -``` -dlpx-app-gate/ -└── appliance/src/.../fluent.conf.stg ← adds - and data_type metric in -``` +The feature is **off by default**. Existing customers are unaffected until explicitly enabled. From 711cb945a269147ba0db9cdf25547e0ecd17ed65 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:32:08 +0530 Subject: [PATCH 09/17] DLPX-100030 docs: expand flow explanation, remove open questions section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the doc crisp but spell out the end-to-end flow in 5 numbered steps with context a reader without background can follow. Drop the open questions section — those are tracked separately. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 85 +++++++++++++----------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md index 1b5ca04..9cd5c33 100644 --- a/docs/fluentd-metric-forwarding-design.md +++ b/docs/fluentd-metric-forwarding-design.md @@ -6,78 +6,89 @@ ## Problem -When a DE is added to DCT, Java analytics is disabled to avoid duplication. This silently stops metric forwarding to customer Splunk / Datadog — they receive nothing. +Delphix customers can forward engine health metrics (CPU, Disk, Network, NFS, iSCSI, TCP) to their own monitoring tools (Splunk, Datadog) through a service called **Fluentd** that already runs on every engine. Historically, Java code on the engine collected those metrics and pushed them to Fluentd. + +When a Delphix Engine (DE) is added to **DCT** (Data Control Tower), DCT becomes the central collector. To avoid sending the same data twice, Java's metric collection is intentionally turned off on that engine. **The side effect: the push to Fluentd stops too.** Those customers now see zero metrics in their Splunk / Datadog dashboards even though nothing else changed in their setup. ``` -DE + DCT today: Java OFF → Fluentd → ❌ Splunk (no data) - Telegraf ON → InfluxDB ✅ (but no Fluentd path) +DE + DCT today: Java OFF → Fluentd → ❌ Splunk (no data) + Telegraf ON → InfluxDB ✅ (local only, no path to Fluentd) ``` --- ## Solution -Add a Fluentd output path to Telegraf so DCT-managed engines resume forwarding metrics to customer monitoring systems with **no changes on the customer side**. +Teach **Telegraf** (which is already running on DCT-managed engines) to send the same six metrics to Fluentd in the same shape Java used — so customers' existing Splunk / Datadog dashboards keep working with **no change on their side**. --- -## Data Flow +## How It Flows (end-to-end) ```mermaid flowchart LR - Admin(["Admin\nSetup Screen"]) -->|"enable/disable"| Toggle["FLUENTD_ENABLED\nflag file"] - Toggle -->|controls| T + Admin(["👤 Admin\nSetup Screen\n(enable 3rd-party monitoring)"]) subgraph DE["Delphix Engine"] - T["Telegraf\ncollects metrics"] --> P["Transform\nrenames + rolls up\nto 1 event/min"] --> F["Fluentd\n:9880"] + direction TB + T["① Telegraf\ncollects raw metrics\nevery 10–60 sec"] + P["② Transform step\n• renames to Java-style names\n• rolls up to 1 event / minute\n• adds engine ID + version"] + F["③ Fluentd\nreceives on port 9880\nroutes to customer plugin"] + T --> P --> F end - F --> Customer(["Splunk / Datadog\n← unchanged"]) + Customer(["🖥️ Customer monitoring\nSplunk / Datadog\n(dashboards unchanged)"]) + + Admin -->|"flips the switch\n(perf_fluentd enable)"| DE + F --> Customer + + style DE fill:#f0f4ff,stroke:#4466cc + style P fill:#fff8e0,stroke:#cc9900 ``` ---- +**Step by step:** -## What Customers Receive (unchanged from Java) +1. **Admin turns it on once** — in the Delphix setup screen, they save Splunk / Datadog credentials. Internally, this calls `perf_fluentd enable`, which creates a flag file and restarts Telegraf. From then on, the engine forwards automatically. -| Field | Value | -|---|---| -| `metric_name` | `system.cpu.util.pct`, `system.net.total.bytes`, `system.disk.ops.count`, `system.nfs.ops.count`, `system.iscsi.ops.count`, `system.tcp.total.bytes` | -| `_value` | Numeric value for that metric | -| `systemUniqueId` | Engine UUID | -| `systemVersion` | Engine version | -| `event` | `"metric"` | -| Frequency | 1 event per metric per minute | -| Protocol | HTTP POST → Fluentd :9880 (was TCP :24224) | +2. **Telegraf collects** — Telegraf is already gathering CPU usage, network bytes, disk operations, etc. every 10–60 seconds. This step changes nothing; the data was already being collected and stored in InfluxDB. + +3. **Transform** — a Starlark script inside Telegraf picks six of those raw metrics, renames them to the exact names the customer's dashboards expect (`system.cpu.util.pct`, `system.net.total.bytes`, etc.), rolls them up to one event per minute, and tags each with the engine UUID and version. This is the "backward compatibility" layer that keeps the customer's dashboards working. + +4. **Send to Fluentd** — Telegraf POSTs those tidy events to Fluentd on localhost port 9880. Fluentd is already running on the engine and already has routing rules (``) that forward anything on this channel to whichever third-party plugin the customer configured. **Those routing rules do not change.** + +5. **Customer's dashboards light up** — Splunk / Datadog receive events with the same metric names, same values, and same identifying fields they always did. The customer sees no difference. --- -## What's New (3 files) +## What Customers Receive (same as Java) -| File | What it does | +| Field | Value | |---|---| -| `perf_fluentd` | CLI: `enable` / `disable` / `status` | -| `telegraf.outputs.fluentd` | HTTP output config (static) | -| `telegraf.processors.fluentd` | Starlark: rename metrics + compute 60s deltas | +| `metric_name` | one of: `system.cpu.util.pct` · `system.net.total.bytes` · `system.disk.ops.count` · `system.nfs.ops.count` · `system.iscsi.ops.count` · `system.tcp.total.bytes` | +| `_value` | numeric value for that metric | +| `systemUniqueId` | engine UUID (lets customers filter by engine in multi-engine setups) | +| `systemVersion` | engine software version | +| `event` | `"metric"` | +| **Frequency** | 1 event per metric per minute | +| **Transport** | HTTP POST → Fluentd `:9880` (replaces old Fluentd Forward on `:24224`) | --- -## Open Questions — Need Answers Before GA +## What's New in the Engine (3 files) -| # | Question | Who | Risk | -|---|---|---|---| -| Q1 | Does `_time` in **seconds** (not ms like Java) break Splunk mstats? | Seb | High | -| Q2 | Does the HEC nested structure `{"fields":{...}}` work with existing customer Fluentd plugins? | Seb | High | -| Q3 | Does `systemVersion` as a full build string break any dashboard filter? | Seb | Medium | -| Q4 | Is `system.tcp.total.bytes` from connstat semantically equivalent to Java's value? | Eng | Medium | -| Q5 | Can we test NFS/iSCSI metrics on an engine with active traffic before GA? | Eng | Medium | +| File | Role | +|---|---| +| `perf_fluentd` | Command-line switch: `enable` / `disable` / `status` | +| `telegraf.outputs.fluentd` | Tells Telegraf where to POST the events | +| `telegraf.processors.fluentd` | The Starlark transform (renaming + rollup) | --- ## Recommendation -| Segment | Today | Action | +| Customer segment | Today | What to do | |---|---|---| -| **DE + DCT** | ❌ Broken — no data in Splunk | Unblock: answer Q1 + Q2 this sprint, enable | -| **DE standalone** | ✅ Java still working | Hold: keep Java, enable Telegraf path only after Q1–Q5 answered | +| **DE + DCT** | ❌ Broken — zero data reaching Splunk | **Unblock first** — this fix resolves it | +| **DE standalone** | ✅ Java still forwarding | Leave Java on until the Telegraf path is proven in the DCT segment | -The feature is **off by default**. Existing customers are unaffected until explicitly enabled. +The feature ships **off by default**. No existing customer is affected until the admin explicitly enables it from the setup screen. From 256e46f172e327cdbab22a20cb524e2701798cb5 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:35:57 +0530 Subject: [PATCH 10/17] DLPX-100030 docs: ground design in DCT-perf-analytics context, cite Confluence + Slack Explain the broader initiative (Telegraf as single collector for DCT engine dashboards), why that initiative disables Java (and silently kills Splunk forwarding), what we're building to restore it, the backward-compat matrix, the one acceptable compromise (data sub-counters), and per-customer migration cost. Links to the DCT performance analytics design page and the Slack thread where the trade-offs were negotiated with Seb. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 150 +++++++++++++++++------ 1 file changed, 110 insertions(+), 40 deletions(-) diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md index 9cd5c33..8e84115 100644 --- a/docs/fluentd-metric-forwarding-design.md +++ b/docs/fluentd-metric-forwarding-design.md @@ -1,94 +1,164 @@ -# Telegraf → Fluentd Metric Forwarding +# Telegraf → Fluentd Metric Forwarding (Backward-Compat for DCT-Managed Engines) -**Jira**: [DLPX-100030](https://perforce.atlassian.net/browse/DLPX-100030) · **Status**: Pending review · **Owner**: Dhruvin Shah +**Jira**: [DLPX-100030](https://perforce.atlassian.net/browse/DLPX-100030) · +**Related design**: [Delphix Engine Performance Analytics in DCT](https://perforce.atlassian.net/wiki/spaces/DLXEN/pages/2740027450) · +**Discussion**: [Slack thread with Seb Roy](https://perforce.slack.com/archives/C09JVAKD3JM/p1793111848235599) --- -## Problem +## 1 · Context — What We're Building -Delphix customers can forward engine health metrics (CPU, Disk, Network, NFS, iSCSI, TCP) to their own monitoring tools (Splunk, Datadog) through a service called **Fluentd** that already runs on every engine. Historically, Java code on the engine collected those metrics and pushed them to Fluentd. +A broader initiative — *Delphix Engine Performance Analytics in DCT* — is moving engine-level performance monitoring **into DCT**. In this new architecture: -When a Delphix Engine (DE) is added to **DCT** (Data Control Tower), DCT becomes the central collector. To avoid sending the same data twice, Java's metric collection is intentionally turned off on that engine. **The side effect: the push to Fluentd stops too.** Those customers now see zero metrics in their Splunk / Datadog dashboards even though nothing else changed in their setup. +- A single **Telegraf** process on each Delphix Engine (DE) is the one and only metric collector. +- Telegraf writes to a **local InfluxDB** on the DE. +- DCT UI embeds Grafana dashboards (via a Smart Proxy) that query that InfluxDB for the selected engine. + +```mermaid +flowchart LR + subgraph DE["Delphix Engine"] + TG["Telegraf\n(single collector)"] --> IDB[("InfluxDB")] + end + IDB --> SP["DCT Smart Proxy"] --> GR["Grafana\n(embedded in DCT UI)"] --> User(["👤 DCT user"]) +``` + +**Side effect of making Telegraf the single collector:** when a DE is added to DCT, the engine's old Java analytics stack (`AnalyticsStorageImpl`, `pushInsightMetric`, etc.) is **intentionally disabled** to avoid duplicate data in InfluxDB and the DCT path. + +--- + +## 2 · The Problem We're Solving Here + +The Java stack that we switch off also happened to be the path by which engines forwarded six KPI metrics to **customer-owned monitoring tools** (Splunk, Datadog, Dynatrace) via a local **Fluentd** service. ``` -DE + DCT today: Java OFF → Fluentd → ❌ Splunk (no data) - Telegraf ON → InfluxDB ✅ (local only, no path to Fluentd) +Standalone DE today : Java ON → Fluentd → ✅ Splunk / Datadog (works) +DE + DCT today : Java OFF → Fluentd → ❌ nothing (silently broken) ``` +Customers who use third-party monitoring and move their engine to DCT **lose that integration**. There is no replacement unless we build one. + +This document covers **only that gap** — the Telegraf → Fluentd forwarding path. + --- -## Solution +## 3 · What We Have vs What We Need -Teach **Telegraf** (which is already running on DCT-managed engines) to send the same six metrics to Fluentd in the same shape Java used — so customers' existing Splunk / Datadog dashboards keep working with **no change on their side**. +| | Standalone DE (today) | DCT-managed DE (today) | DCT-managed DE (after this change) | +|---|---|---|---| +| Local Telegraf | ✅ running | ✅ running | ✅ running | +| InfluxDB | ✅ | ✅ | ✅ | +| DCT dashboards | n/a | ✅ (via Smart Proxy) | ✅ (unchanged) | +| **Java → Fluentd** | ✅ working | ❌ **disabled** | ❌ disabled (intentional) | +| **Telegraf → Fluentd** | — | — | ✅ **this feature** | +| Splunk / Datadog receiving data | ✅ | ❌ | ✅ | --- -## How It Flows (end-to-end) +## 4 · How The New Path Works (end-to-end) ```mermaid flowchart LR - Admin(["👤 Admin\nSetup Screen\n(enable 3rd-party monitoring)"]) + Admin(["👤 Admin\nSetup Screen\nenables 3rd-party\nmonitoring"]) subgraph DE["Delphix Engine"] direction TB - T["① Telegraf\ncollects raw metrics\nevery 10–60 sec"] - P["② Transform step\n• renames to Java-style names\n• rolls up to 1 event / minute\n• adds engine ID + version"] - F["③ Fluentd\nreceives on port 9880\nroutes to customer plugin"] + T["① Telegraf\ncollects raw metrics\n(CPU, Net, Disk,\nNFS, iSCSI, TCP)"] + P["② Transform step\n• renames to Java-style names\n• rolls up to 1 event / min\n• adds engine UUID + version"] + F["③ Fluentd on :9880\nroutes to customer plugin\nvia existing match rules"] T --> P --> F end - Customer(["🖥️ Customer monitoring\nSplunk / Datadog\n(dashboards unchanged)"]) + Customer(["🖥️ Splunk / Datadog / Dynatrace\n(dashboards unchanged)"]) - Admin -->|"flips the switch\n(perf_fluentd enable)"| DE + Admin -->|"perf_fluentd enable\n(one-time)"| DE F --> Customer style DE fill:#f0f4ff,stroke:#4466cc style P fill:#fff8e0,stroke:#cc9900 ``` -**Step by step:** +**Step by step for a reader unfamiliar with the stack:** -1. **Admin turns it on once** — in the Delphix setup screen, they save Splunk / Datadog credentials. Internally, this calls `perf_fluentd enable`, which creates a flag file and restarts Telegraf. From then on, the engine forwards automatically. +1. **Admin flips the switch once.** In the Delphix setup screen, the admin saves their Splunk / Datadog credentials. Behind the scenes, this calls a small shell script `perf_fluentd enable` that creates a flag file and restarts Telegraf. -2. **Telegraf collects** — Telegraf is already gathering CPU usage, network bytes, disk operations, etc. every 10–60 seconds. This step changes nothing; the data was already being collected and stored in InfluxDB. +2. **Telegraf keeps collecting as it already does.** It gathers CPU usage, network bytes, disk I/O, etc. every 10–60 seconds and writes to local InfluxDB. **No change to that behaviour.** The data for DCT Grafana dashboards continues flowing exactly as before. -3. **Transform** — a Starlark script inside Telegraf picks six of those raw metrics, renames them to the exact names the customer's dashboards expect (`system.cpu.util.pct`, `system.net.total.bytes`, etc.), rolls them up to one event per minute, and tags each with the engine UUID and version. This is the "backward compatibility" layer that keeps the customer's dashboards working. +3. **A small transform runs inside Telegraf** (only when the flag file exists). It picks the six KPIs that Java used to send, renames them to the exact names existing customer dashboards expect (`system.cpu.util.pct`, `system.net.total.bytes`, …), rolls them up to one event per minute, and tags each with the engine UUID and version so customers can filter by engine in multi-engine Splunk setups. -4. **Send to Fluentd** — Telegraf POSTs those tidy events to Fluentd on localhost port 9880. Fluentd is already running on the engine and already has routing rules (``) that forward anything on this channel to whichever third-party plugin the customer configured. **Those routing rules do not change.** +4. **Telegraf POSTs those tidy events to Fluentd** on `localhost:9880` (HTTP). Fluentd is already running and already has a `` rule that routes to the customer's chosen plugin — Splunk HEC, Datadog, or Dynatrace. **Those routing rules do not change.** -5. **Customer's dashboards light up** — Splunk / Datadog receive events with the same metric names, same values, and same identifying fields they always did. The customer sees no difference. +5. **Customer dashboards light up again** with the same metric names, same values, same identifying fields they had before DCT. --- -## What Customers Receive (same as Java) +## 5 · Backward Compatibility — What Customers See + +To preserve existing Splunk searches, Datadog dashboards and Dynatrace alerts, we match the Java event shape **almost exactly** (see [Slack thread](https://perforce.slack.com/archives/C09JVAKD3JM/p1793111848235599) for the detailed negotiation with Seb): -| Field | Value | +| Field | Java (old) | Telegraf (new) | Compatible? | +|---|---|---|---| +| `metric_name` | `system.cpu.util.pct` etc. | **same** (renamed in transform) | ✅ | +| `_value` | numeric | **same** (semantic value preserved) | ✅ | +| `_time` | epoch | epoch | ✅ | +| `event` | `"metric"` | `"metric"` | ✅ | +| `systemUniqueId` | engine UUID | engine UUID (via `get-system-uuid`) | ✅ | +| `systemVersion` | engine version | engine version (via `get-appliance-version`) | ✅ | +| Frequency | 1 / min / metric | 1 / min / metric | ✅ | +| **`data` sub-counters** | `{idle, user, kernel}` | **not sent** — Telegraf doesn't expose raw kstat values | ❌ | + +**The one compromise:** customers who built Splunk searches on the `data.idle` / `data.user` / `data.kernel` sub-fields lose those. Everything else survives the migration unchanged. + +### Customer migration cost by segment + +| Segment | What they must do | |---|---| -| `metric_name` | one of: `system.cpu.util.pct` · `system.net.total.bytes` · `system.disk.ops.count` · `system.nfs.ops.count` · `system.iscsi.ops.count` · `system.tcp.total.bytes` | -| `_value` | numeric value for that metric | -| `systemUniqueId` | engine UUID (lets customers filter by engine in multi-engine setups) | -| `systemVersion` | engine software version | -| `event` | `"metric"` | -| **Frequency** | 1 event per metric per minute | -| **Transport** | HTTP POST → Fluentd `:9880` (replaces old Fluentd Forward on `:24224`) | +| **Splunk HEC users** | Nothing — the companion template change ([app-gate PR #5043](https://github.com/delphix/dlpx-app-gate/pull/5043)) ships the new `` block in the standard `fluent.conf.stg` automatically | +| **Datadog users** (×2 known) | Add a 5-line `` stanza to their custom Fluentd template | +| **Dynatrace user** (×1 known) | Same as Datadog — 5-line stanza | --- -## What's New in the Engine (3 files) +## 6 · User Controls + +The forwarding is **off by default**. The admin controls it from the Delphix setup screen; nothing happens on a customer engine until they opt in. + +```mermaid +flowchart LR + A(["Admin saves\n3rd-party credentials"]) --> B["app-gate calls\nperf_fluentd enable"] + B --> C["creates flag file\nFLUENTD_ENABLED"] + C --> D["restarts Telegraf\n(reads flag, loads transform)"] + D --> E(["✅ forwarding on"]) + + F(["Admin removes\ncredentials"]) --> G["app-gate calls\nperf_fluentd disable"] + G --> H["removes flag file"] + H --> I["restarts Telegraf\n(no transform, no HTTP output)"] + I --> J(["⛔ forwarding off\nInfluxDB & DCT unaffected"]) + + style E fill:#ddffdd,stroke:#44aa44 + style J fill:#ffdddd,stroke:#aa4444 +``` + +The flag file (`/etc/telegraf/FLUENTD_ENABLED`) is the single source of truth. `perf_fluentd status` reports its state. + +--- + +## 7 · What's New in the Engine (3 small files) | File | Role | |---|---| -| `perf_fluentd` | Command-line switch: `enable` / `disable` / `status` | -| `telegraf.outputs.fluentd` | Tells Telegraf where to POST the events | -| `telegraf.processors.fluentd` | The Starlark transform (renaming + rollup) | +| `perf_fluentd` | Shell CLI: `enable` / `disable` / `status` | +| `telegraf.outputs.fluentd` | Tells Telegraf where to POST (localhost:9880, splunkmetric format, 60 s flush) | +| `telegraf.processors.fluentd` | Starlark script that renames metrics, computes deltas, rolls up to 1 event / min, adds engine identity | + +Existing `delphix-telegraf-service` is modified to include these files when the flag is set. --- -## Recommendation +## 8 · Recommendation -| Customer segment | Today | What to do | +| Segment | Status today | Action | |---|---|---| -| **DE + DCT** | ❌ Broken — zero data reaching Splunk | **Unblock first** — this fix resolves it | -| **DE standalone** | ✅ Java still forwarding | Leave Java on until the Telegraf path is proven in the DCT segment | +| **DE + DCT** | Silently broken — zero data in Splunk / Datadog | **Enable this path** — it is the only way to restore their integration | +| **Standalone DE** | Java still forwarding — unaffected | Keep Java on; enable the Telegraf path only once it's proven in the DCT segment | -The feature ships **off by default**. No existing customer is affected until the admin explicitly enables it from the setup screen. +Shipping this feature does not affect any customer who doesn't flip the switch. For DCT-managed engines where the integration is already broken, this is the restore mechanism. From 0e326467c8384577b22081fe633409999bc8df6e Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Wed, 28 Oct 2026 22:40:30 +0530 Subject: [PATCH 11/17] DLPX-100030 docs: add reference PRs (perf-diag #135, app-gate #5043) for review Reference-PRs table at the top of the doc links to the sample implementation alongside the design, so reviewers can see the exact Starlark transform, CLI, Fluentd template, and FluentServiceImpl lifecycle wiring that this doc describes. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md index 8e84115..f0fe5d1 100644 --- a/docs/fluentd-metric-forwarding-design.md +++ b/docs/fluentd-metric-forwarding-design.md @@ -4,6 +4,13 @@ **Related design**: [Delphix Engine Performance Analytics in DCT](https://perforce.atlassian.net/wiki/spaces/DLXEN/pages/2740027450) · **Discussion**: [Slack thread with Seb Roy](https://perforce.slack.com/archives/C09JVAKD3JM/p1793111848235599) +**Reference PRs** (sample implementation for review alongside this doc): + +| PR | Repo | What it does | +|---|---|---| +| [#135](https://github.com/delphix/performance-diagnostics/pull/135) | `performance-diagnostics` | Telegraf HTTP output, Starlark transform (name mapping + delta rollup), engine identity injection, `perf_fluentd` CLI | +| [#5043](https://github.com/delphix/dlpx-app-gate/pull/5043) | `dlpx-app-gate` | Removes the Java `pushInsightMetric` path, adds `` to `fluent.conf.stg`, wires `perf_fluentd enable/disable` into the Fluentd lifecycle (`FluentServiceImpl`) | + --- ## 1 · Context — What We're Building From a1857635b26caf3e8923a4bcef3d714ab2665268 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Thu, 29 Oct 2026 10:56:56 +0530 Subject: [PATCH 12/17] DLPX-100030 docs: remove design doc from PR (lives on Confluence instead) Design document is now the source-of-truth artefact at: https://perforce.atlassian.net/wiki/spaces/DLXEN/pages/3722117178 Keeping an in-repo copy risks drift; the Confluence page is linked from the PR description. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/fluentd-metric-forwarding-design.md | 171 ----------------------- 1 file changed, 171 deletions(-) delete mode 100644 docs/fluentd-metric-forwarding-design.md diff --git a/docs/fluentd-metric-forwarding-design.md b/docs/fluentd-metric-forwarding-design.md deleted file mode 100644 index f0fe5d1..0000000 --- a/docs/fluentd-metric-forwarding-design.md +++ /dev/null @@ -1,171 +0,0 @@ -# Telegraf → Fluentd Metric Forwarding (Backward-Compat for DCT-Managed Engines) - -**Jira**: [DLPX-100030](https://perforce.atlassian.net/browse/DLPX-100030) · -**Related design**: [Delphix Engine Performance Analytics in DCT](https://perforce.atlassian.net/wiki/spaces/DLXEN/pages/2740027450) · -**Discussion**: [Slack thread with Seb Roy](https://perforce.slack.com/archives/C09JVAKD3JM/p1793111848235599) - -**Reference PRs** (sample implementation for review alongside this doc): - -| PR | Repo | What it does | -|---|---|---| -| [#135](https://github.com/delphix/performance-diagnostics/pull/135) | `performance-diagnostics` | Telegraf HTTP output, Starlark transform (name mapping + delta rollup), engine identity injection, `perf_fluentd` CLI | -| [#5043](https://github.com/delphix/dlpx-app-gate/pull/5043) | `dlpx-app-gate` | Removes the Java `pushInsightMetric` path, adds `` to `fluent.conf.stg`, wires `perf_fluentd enable/disable` into the Fluentd lifecycle (`FluentServiceImpl`) | - ---- - -## 1 · Context — What We're Building - -A broader initiative — *Delphix Engine Performance Analytics in DCT* — is moving engine-level performance monitoring **into DCT**. In this new architecture: - -- A single **Telegraf** process on each Delphix Engine (DE) is the one and only metric collector. -- Telegraf writes to a **local InfluxDB** on the DE. -- DCT UI embeds Grafana dashboards (via a Smart Proxy) that query that InfluxDB for the selected engine. - -```mermaid -flowchart LR - subgraph DE["Delphix Engine"] - TG["Telegraf\n(single collector)"] --> IDB[("InfluxDB")] - end - IDB --> SP["DCT Smart Proxy"] --> GR["Grafana\n(embedded in DCT UI)"] --> User(["👤 DCT user"]) -``` - -**Side effect of making Telegraf the single collector:** when a DE is added to DCT, the engine's old Java analytics stack (`AnalyticsStorageImpl`, `pushInsightMetric`, etc.) is **intentionally disabled** to avoid duplicate data in InfluxDB and the DCT path. - ---- - -## 2 · The Problem We're Solving Here - -The Java stack that we switch off also happened to be the path by which engines forwarded six KPI metrics to **customer-owned monitoring tools** (Splunk, Datadog, Dynatrace) via a local **Fluentd** service. - -``` -Standalone DE today : Java ON → Fluentd → ✅ Splunk / Datadog (works) -DE + DCT today : Java OFF → Fluentd → ❌ nothing (silently broken) -``` - -Customers who use third-party monitoring and move their engine to DCT **lose that integration**. There is no replacement unless we build one. - -This document covers **only that gap** — the Telegraf → Fluentd forwarding path. - ---- - -## 3 · What We Have vs What We Need - -| | Standalone DE (today) | DCT-managed DE (today) | DCT-managed DE (after this change) | -|---|---|---|---| -| Local Telegraf | ✅ running | ✅ running | ✅ running | -| InfluxDB | ✅ | ✅ | ✅ | -| DCT dashboards | n/a | ✅ (via Smart Proxy) | ✅ (unchanged) | -| **Java → Fluentd** | ✅ working | ❌ **disabled** | ❌ disabled (intentional) | -| **Telegraf → Fluentd** | — | — | ✅ **this feature** | -| Splunk / Datadog receiving data | ✅ | ❌ | ✅ | - ---- - -## 4 · How The New Path Works (end-to-end) - -```mermaid -flowchart LR - Admin(["👤 Admin\nSetup Screen\nenables 3rd-party\nmonitoring"]) - - subgraph DE["Delphix Engine"] - direction TB - T["① Telegraf\ncollects raw metrics\n(CPU, Net, Disk,\nNFS, iSCSI, TCP)"] - P["② Transform step\n• renames to Java-style names\n• rolls up to 1 event / min\n• adds engine UUID + version"] - F["③ Fluentd on :9880\nroutes to customer plugin\nvia existing match rules"] - T --> P --> F - end - - Customer(["🖥️ Splunk / Datadog / Dynatrace\n(dashboards unchanged)"]) - - Admin -->|"perf_fluentd enable\n(one-time)"| DE - F --> Customer - - style DE fill:#f0f4ff,stroke:#4466cc - style P fill:#fff8e0,stroke:#cc9900 -``` - -**Step by step for a reader unfamiliar with the stack:** - -1. **Admin flips the switch once.** In the Delphix setup screen, the admin saves their Splunk / Datadog credentials. Behind the scenes, this calls a small shell script `perf_fluentd enable` that creates a flag file and restarts Telegraf. - -2. **Telegraf keeps collecting as it already does.** It gathers CPU usage, network bytes, disk I/O, etc. every 10–60 seconds and writes to local InfluxDB. **No change to that behaviour.** The data for DCT Grafana dashboards continues flowing exactly as before. - -3. **A small transform runs inside Telegraf** (only when the flag file exists). It picks the six KPIs that Java used to send, renames them to the exact names existing customer dashboards expect (`system.cpu.util.pct`, `system.net.total.bytes`, …), rolls them up to one event per minute, and tags each with the engine UUID and version so customers can filter by engine in multi-engine Splunk setups. - -4. **Telegraf POSTs those tidy events to Fluentd** on `localhost:9880` (HTTP). Fluentd is already running and already has a `` rule that routes to the customer's chosen plugin — Splunk HEC, Datadog, or Dynatrace. **Those routing rules do not change.** - -5. **Customer dashboards light up again** with the same metric names, same values, same identifying fields they had before DCT. - ---- - -## 5 · Backward Compatibility — What Customers See - -To preserve existing Splunk searches, Datadog dashboards and Dynatrace alerts, we match the Java event shape **almost exactly** (see [Slack thread](https://perforce.slack.com/archives/C09JVAKD3JM/p1793111848235599) for the detailed negotiation with Seb): - -| Field | Java (old) | Telegraf (new) | Compatible? | -|---|---|---|---| -| `metric_name` | `system.cpu.util.pct` etc. | **same** (renamed in transform) | ✅ | -| `_value` | numeric | **same** (semantic value preserved) | ✅ | -| `_time` | epoch | epoch | ✅ | -| `event` | `"metric"` | `"metric"` | ✅ | -| `systemUniqueId` | engine UUID | engine UUID (via `get-system-uuid`) | ✅ | -| `systemVersion` | engine version | engine version (via `get-appliance-version`) | ✅ | -| Frequency | 1 / min / metric | 1 / min / metric | ✅ | -| **`data` sub-counters** | `{idle, user, kernel}` | **not sent** — Telegraf doesn't expose raw kstat values | ❌ | - -**The one compromise:** customers who built Splunk searches on the `data.idle` / `data.user` / `data.kernel` sub-fields lose those. Everything else survives the migration unchanged. - -### Customer migration cost by segment - -| Segment | What they must do | -|---|---| -| **Splunk HEC users** | Nothing — the companion template change ([app-gate PR #5043](https://github.com/delphix/dlpx-app-gate/pull/5043)) ships the new `` block in the standard `fluent.conf.stg` automatically | -| **Datadog users** (×2 known) | Add a 5-line `` stanza to their custom Fluentd template | -| **Dynatrace user** (×1 known) | Same as Datadog — 5-line stanza | - ---- - -## 6 · User Controls - -The forwarding is **off by default**. The admin controls it from the Delphix setup screen; nothing happens on a customer engine until they opt in. - -```mermaid -flowchart LR - A(["Admin saves\n3rd-party credentials"]) --> B["app-gate calls\nperf_fluentd enable"] - B --> C["creates flag file\nFLUENTD_ENABLED"] - C --> D["restarts Telegraf\n(reads flag, loads transform)"] - D --> E(["✅ forwarding on"]) - - F(["Admin removes\ncredentials"]) --> G["app-gate calls\nperf_fluentd disable"] - G --> H["removes flag file"] - H --> I["restarts Telegraf\n(no transform, no HTTP output)"] - I --> J(["⛔ forwarding off\nInfluxDB & DCT unaffected"]) - - style E fill:#ddffdd,stroke:#44aa44 - style J fill:#ffdddd,stroke:#aa4444 -``` - -The flag file (`/etc/telegraf/FLUENTD_ENABLED`) is the single source of truth. `perf_fluentd status` reports its state. - ---- - -## 7 · What's New in the Engine (3 small files) - -| File | Role | -|---|---| -| `perf_fluentd` | Shell CLI: `enable` / `disable` / `status` | -| `telegraf.outputs.fluentd` | Tells Telegraf where to POST (localhost:9880, splunkmetric format, 60 s flush) | -| `telegraf.processors.fluentd` | Starlark script that renames metrics, computes deltas, rolls up to 1 event / min, adds engine identity | - -Existing `delphix-telegraf-service` is modified to include these files when the flag is set. - ---- - -## 8 · Recommendation - -| Segment | Status today | Action | -|---|---|---| -| **DE + DCT** | Silently broken — zero data in Splunk / Datadog | **Enable this path** — it is the only way to restore their integration | -| **Standalone DE** | Java still forwarding — unaffected | Keep Java on; enable the Telegraf path only once it's proven in the DCT segment | - -Shipping this feature does not affect any customer who doesn't flip the switch. For DCT-managed engines where the integration is already broken, this is the restore mechanism. From d300780701029ecb9a6aa89eb5fdae80d4f9c334 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Thu, 29 Oct 2026 12:16:32 +0530 Subject: [PATCH 13/17] DLPX-100030 telegraf.outputs.fluentd: use_batch_format=false to prevent silent event loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit splunkmetric_hec_routing serializes a batch as raw-concatenated JSON objects with no delimiter or newline. Fluentd's JSON parser consumes only the first object per POST, silently dropping the rest — ~83% loss once all six compat metrics are flowing in production. use_batch_format=false serializes one metric per POST (6 small localhost requests per minute, negligible overhead) so every event lands. Verified on dbs-test.dlpxdc.co and ev-dev.dlpxdc.co: each POST body now contains exactly 1 top-level JSON object, all compat metric_names observed. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/telegraf.outputs.fluentd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd index ca9437c..061462b 100644 --- a/telegraf/telegraf.outputs.fluentd +++ b/telegraf/telegraf.outputs.fluentd @@ -8,6 +8,8 @@ method = "POST" data_format = "splunkmetric" splunkmetric_hec_routing = true + # One POST per metric: batched splunkmetric emits concatenated JSON with no delimiter, which fluentd's JSON parser silently truncates to the first object. + use_batch_format = false flush_interval = "60s" timeout = "10s" # Buffer a few minutes of compat metrics so a brief fluentd restart doesn't From a0b2d9b4d9e1e6971c7802ff4dbd80c8990b34e0 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Thu, 29 Oct 2026 16:26:29 +0530 Subject: [PATCH 14/17] DLPX-100030 telegraf.processors.fluentd: enrich data.* breakdown fields Each compat metric now carries rich data.* breakdown alongside _value, closer to Java's pushInsightMetric payload shape: system.cpu.util.pct 7 fields: usage_user/system/idle/iowait/ irq/softirq/steal (percentages) system.net.total.bytes 4 fields: inBytes, outBytes, inPackets, outPackets system.disk.ops.count 6 fields: reads, writes, bytesRead, bytesWritten, readTime, writeTime system.nfs.ops.count 6 fields: readOps, writeOps, read/writeLatency, read/writeThroughput system.iscsi.ops.count 6 fields: same shape as nfs system.tcp.total.bytes 8 fields: inBytes, outBytes, retransmittedSegs, sendWindowSize, congestionWindowSize, receiveWindowSize, roundTripTime, connections Note: round() triggers a Telegraf 1.40.2 starlark resolver bug ("undefined: state" at compile time). We serialise numeric data via plain str() instead, which is why percentages and rates show full float precision. Verified end-to-end on ev-dev.dlpxdc.co and dbs-test.dlpxdc.co: processor loads cleanly, all data.* fields appear in captured HEC events alongside metric_name/_value/systemUniqueId/systemVersion. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/telegraf.processors.fluentd | 193 +++++++++++++++++++++------ 1 file changed, 152 insertions(+), 41 deletions(-) diff --git a/telegraf/telegraf.processors.fluentd b/telegraf/telegraf.processors.fluentd index 094541a..dd6a2d2 100644 --- a/telegraf/telegraf.processors.fluentd +++ b/telegraf/telegraf.processors.fluentd @@ -1,30 +1,10 @@ # Starlark processor — maps Telegraf measurements to Java InsightLogger metric -# names (system.*) with semantics matching Java's pushInsightMetric: +# names (system.*) with semantics matching Java's pushInsightMetric. +# Attaches rich data.* breakdown fields alongside _value (Telegraf-native names). # -# system.cpu.util.pct — % CPU busy in the window (gauge: last sample / 60s) -# system.net.total.bytes — DELTA bytes (recv+sent) over the window, all ifaces -# system.disk.ops.count — DELTA ops (read+write) over the window, all devices -# system.nfs.ops.count — ops (read+write) over the window across directions -# system.iscsi.ops.count — ops (read+write) over the window across directions -# system.tcp.total.bytes — DELTA bytes (in+out) over the window, all endpoints +# Note: round() triggers a Telegraf 1.40.2 starlark resolver bug ("undefined: +# state"). We emit numeric values via plain str() instead. # -# Java polled /proc/net/dev, /proc/diskstats, etc. at ~60s cadence and sent -# (current_cumulative - previous_cumulative). This processor does the same: -# - net / diskio / tcp_stats fields are cumulative kernel counters; -# we subtract the previous poll's value per-tag to recover the per-sample -# delta, then roll up per-sample deltas into a 60s bucket. -# - estat_nfs/_iscsi expose iops(/s), a rate averaged over the 10s collection -# window; ops-in-window = iops * 10, summed into the 60s bucket. -# - cpu usage_idle is already a percentage over the sample window — we keep -# the last sample's value in the 60s bucket (Java's gauge semantics). -# -# All bucketing happens here (no aggregator plugin) because Telegraf's -# basicstats aggregator appends "_sum" to field names, which would corrupt the -# splunkmetric-derived metric_name. Bucket emission is one 60s cycle behind -# (previous bucket emitted when the first sample of the next bucket arrives). -# -# systemUniqueId / systemVersion are scoped to the compat metrics only (set -# here, not as global_tags) so InfluxDB cardinality is unaffected. # ${DELPHIX_SYSTEM_UUID} / ${DELPHIX_SYSTEM_VERSION} are substituted at TOML # parse time from env vars exported by delphix-telegraf-service. # @@ -91,7 +71,7 @@ def apply(metric): ts = metric.time bucket = ts // BUCKET_NS - # system.cpu.util.pct — gauge; keep last cpu-total sample per minute. + # system.cpu.util.pct — data.* carries all 7 cpu-time breakdown fields. if name == "cpu": if metric.tags.get("cpu") != "cpu-total": return [metric] @@ -99,58 +79,189 @@ def apply(metric): return [metric] out = [metric] out = out + flush_gauge("cpu", "system.cpu.util", "pct", bucket) - state["cpu_last"] = 100.0 - float(metric.fields["usage_idle"]) + if len(out) > 1: + out[1].tags["data.usage_user"] = str(state.get("cpu_d_usr", 0.0)) + out[1].tags["data.usage_system"] = str(state.get("cpu_d_sys", 0.0)) + out[1].tags["data.usage_idle"] = str(state.get("cpu_d_idl", 0.0)) + out[1].tags["data.usage_iowait"] = str(state.get("cpu_d_iow", 0.0)) + out[1].tags["data.usage_irq"] = str(state.get("cpu_d_irq", 0.0)) + out[1].tags["data.usage_softirq"] = str(state.get("cpu_d_sir", 0.0)) + out[1].tags["data.usage_steal"] = str(state.get("cpu_d_stl", 0.0)) + state["cpu_last"] = 100.0 - float(metric.fields["usage_idle"]) + state["cpu_d_usr"] = float(metric.fields.get("usage_user", 0)) + state["cpu_d_sys"] = float(metric.fields.get("usage_system", 0)) + state["cpu_d_idl"] = float(metric.fields.get("usage_idle", 0)) + state["cpu_d_iow"] = float(metric.fields.get("usage_iowait", 0)) + state["cpu_d_irq"] = float(metric.fields.get("usage_irq", 0)) + state["cpu_d_sir"] = float(metric.fields.get("usage_softirq", 0)) + state["cpu_d_stl"] = float(metric.fields.get("usage_steal", 0)) return out - # system.nfs.ops.count — sum ops across read/write rows and 10s windows. + # system.nfs.ops.count — data.* includes read/write ops + latency + throughput. if name == "estat_nfs": if "iops(/s)" not in metric.fields: return [metric] + iops = float(metric.fields["iops(/s)"]) * 10.0 + latency = float(metric.fields.get("avg latency(us)", 0)) + throughput = float(metric.fields.get("throughput(k/s)", 0)) + direction = metric.tags.get("name", "") out = [metric] out = out + flush_counter("nfs", "system.nfs.ops", "count", bucket) - state["nfs_acc"] = state.get("nfs_acc", 0.0) + float(metric.fields["iops(/s)"]) * 10.0 + if len(out) > 1: + out[1].tags["data.readOps"] = str(int(state.get("nfs_d_r", 0.0))) + out[1].tags["data.writeOps"] = str(int(state.get("nfs_d_w", 0.0))) + out[1].tags["data.readLatency"] = str(state.get("nfs_rl", 0.0)) + out[1].tags["data.writeLatency"] = str(state.get("nfs_wl", 0.0)) + out[1].tags["data.readThroughput"] = str(state.get("nfs_rt", 0.0)) + out[1].tags["data.writeThroughput"] = str(state.get("nfs_wt", 0.0)) + state["nfs_d_r"] = 0.0 + state["nfs_d_w"] = 0.0 + state["nfs_acc"] = state.get("nfs_acc", 0.0) + iops + if direction == "read": + state["nfs_d_r"] = state.get("nfs_d_r", 0.0) + iops + state["nfs_rl"] = latency + state["nfs_rt"] = throughput + else: + state["nfs_d_w"] = state.get("nfs_d_w", 0.0) + iops + state["nfs_wl"] = latency + state["nfs_wt"] = throughput return out # system.iscsi.ops.count — same shape as nfs. if name == "estat_iscsi": if "iops(/s)" not in metric.fields: return [metric] + iops = float(metric.fields["iops(/s)"]) * 10.0 + latency = float(metric.fields.get("avg latency(us)", 0)) + throughput = float(metric.fields.get("throughput(k/s)", 0)) + direction = metric.tags.get("name", "") out = [metric] out = out + flush_counter("iscsi", "system.iscsi.ops", "count", bucket) - state["iscsi_acc"] = state.get("iscsi_acc", 0.0) + float(metric.fields["iops(/s)"]) * 10.0 + if len(out) > 1: + out[1].tags["data.readOps"] = str(int(state.get("isc_d_r", 0.0))) + out[1].tags["data.writeOps"] = str(int(state.get("isc_d_w", 0.0))) + out[1].tags["data.readLatency"] = str(state.get("isc_rl", 0.0)) + out[1].tags["data.writeLatency"] = str(state.get("isc_wl", 0.0)) + out[1].tags["data.readThroughput"] = str(state.get("isc_rt", 0.0)) + out[1].tags["data.writeThroughput"] = str(state.get("isc_wt", 0.0)) + state["isc_d_r"] = 0.0 + state["isc_d_w"] = 0.0 + state["iscsi_acc"] = state.get("iscsi_acc", 0.0) + iops + if direction == "read": + state["isc_d_r"] = state.get("isc_d_r", 0.0) + iops + state["isc_rl"] = latency + state["isc_rt"] = throughput + else: + state["isc_d_w"] = state.get("isc_d_w", 0.0) + iops + state["isc_wl"] = latency + state["isc_wt"] = throughput return out - # system.net.total.bytes — per-interface cumulative-counter delta, summed. + # system.net.total.bytes — data.* includes bytes + packets. if name == "net": iface = metric.tags.get("interface", "") recv = int(metric.fields.get("bytes_recv", 0)) sent = int(metric.fields.get("bytes_sent", 0)) - d = cumulative_delta("net_r_" + iface, recv) + cumulative_delta("net_s_" + iface, sent) + recv_pkt = int(metric.fields.get("packets_recv", 0)) + sent_pkt = int(metric.fields.get("packets_sent", 0)) + d_in = cumulative_delta("net_r_" + iface, recv) + d_out = cumulative_delta("net_s_" + iface, sent) + d_ip = cumulative_delta("net_rp_" + iface, recv_pkt) + d_op = cumulative_delta("net_sp_" + iface, sent_pkt) out = [metric] out = out + flush_counter("net", "system.net.total", "bytes", bucket) - state["net_acc"] = state.get("net_acc", 0.0) + float(d) + if len(out) > 1: + out[1].tags["data.inBytes"] = str(int(state.get("net_d_in", 0.0))) + out[1].tags["data.outBytes"] = str(int(state.get("net_d_out", 0.0))) + out[1].tags["data.inPackets"] = str(int(state.get("net_d_ip", 0.0))) + out[1].tags["data.outPackets"] = str(int(state.get("net_d_op", 0.0))) + state["net_d_in"] = 0.0 + state["net_d_out"] = 0.0 + state["net_d_ip"] = 0.0 + state["net_d_op"] = 0.0 + state["net_acc"] = state.get("net_acc", 0.0) + float(d_in + d_out) + state["net_d_in"] = state.get("net_d_in", 0.0) + float(d_in) + state["net_d_out"] = state.get("net_d_out", 0.0) + float(d_out) + state["net_d_ip"] = state.get("net_d_ip", 0.0) + float(d_ip) + state["net_d_op"] = state.get("net_d_op", 0.0) + float(d_op) return out - # system.disk.ops.count — per-device cumulative-counter delta, summed. + # system.disk.ops.count — data.* includes reads/writes + bytes + times. if name == "diskio": dev = metric.tags.get("name", "") - r = int(metric.fields.get("reads", 0)) - w = int(metric.fields.get("writes", 0)) - d = cumulative_delta("disk_r_" + dev, r) + cumulative_delta("disk_w_" + dev, w) + r = int(metric.fields.get("reads", 0)) + w = int(metric.fields.get("writes", 0)) + rb = int(metric.fields.get("read_bytes", 0)) + wb = int(metric.fields.get("write_bytes", 0)) + rt = int(metric.fields.get("read_time", 0)) + wt = int(metric.fields.get("write_time", 0)) + d_r = cumulative_delta("disk_r_" + dev, r) + d_w = cumulative_delta("disk_w_" + dev, w) + d_rb = cumulative_delta("disk_rb_" + dev, rb) + d_wb = cumulative_delta("disk_wb_" + dev, wb) + d_rt = cumulative_delta("disk_rt_" + dev, rt) + d_wt = cumulative_delta("disk_wt_" + dev, wt) out = [metric] out = out + flush_counter("disk", "system.disk.ops", "count", bucket) - state["disk_acc"] = state.get("disk_acc", 0.0) + float(d) + if len(out) > 1: + out[1].tags["data.reads"] = str(int(state.get("dsk_d_r", 0.0))) + out[1].tags["data.writes"] = str(int(state.get("dsk_d_w", 0.0))) + out[1].tags["data.bytesRead"] = str(int(state.get("dsk_d_rb", 0.0))) + out[1].tags["data.bytesWritten"] = str(int(state.get("dsk_d_wb", 0.0))) + out[1].tags["data.readTime"] = str(int(state.get("dsk_d_rt", 0.0))) + out[1].tags["data.writeTime"] = str(int(state.get("dsk_d_wt", 0.0))) + state["dsk_d_r"] = 0.0 + state["dsk_d_w"] = 0.0 + state["dsk_d_rb"] = 0.0 + state["dsk_d_wb"] = 0.0 + state["dsk_d_rt"] = 0.0 + state["dsk_d_wt"] = 0.0 + state["disk_acc"] = state.get("disk_acc", 0.0) + float(d_r + d_w) + state["dsk_d_r"] = state.get("dsk_d_r", 0.0) + float(d_r) + state["dsk_d_w"] = state.get("dsk_d_w", 0.0) + float(d_w) + state["dsk_d_rb"] = state.get("dsk_d_rb", 0.0) + float(d_rb) + state["dsk_d_wb"] = state.get("dsk_d_wb", 0.0) + float(d_wb) + state["dsk_d_rt"] = state.get("dsk_d_rt", 0.0) + float(d_rt) + state["dsk_d_wt"] = state.get("dsk_d_wt", 0.0) + float(d_wt) return out - # system.tcp.total.bytes — per-endpoint cumulative-counter delta, summed. + # system.tcp.total.bytes — data.* includes bytes + retransmits + windows + rtt. if name == "tcp_stats": - key = metric.tags.get("laddr", "") + "|" + metric.tags.get("raddr", "") + "|" + metric.tags.get("service", "") + key = metric.tags.get("laddr", "") + "|" + metric.tags.get("raddr", "") + "|" + metric.tags.get("service", "") inb = int(metric.fields.get("inbytes", 0)) outb = int(metric.fields.get("outbytes", 0)) - d = cumulative_delta("tcp_in_" + key, inb) + cumulative_delta("tcp_out_" + key, outb) + rseg = int(metric.fields.get("retranssegs", 0)) + swnd = int(metric.fields.get("swnd", 0)) + cwnd = int(metric.fields.get("cwnd", 0)) + rwnd = int(metric.fields.get("rwnd", 0)) + rtt = int(metric.fields.get("rtt", 0)) + conn = int(metric.fields.get("connections", 0)) + d_in = cumulative_delta("tcp_in_" + key, inb) + d_out = cumulative_delta("tcp_out_" + key, outb) + d_rs = cumulative_delta("tcp_rs_" + key, rseg) out = [metric] out = out + flush_counter("tcp", "system.tcp.total", "bytes", bucket) - state["tcp_acc"] = state.get("tcp_acc", 0.0) + float(d) + if len(out) > 1: + out[1].tags["data.inBytes"] = str(int(state.get("tcp_d_in", 0.0))) + out[1].tags["data.outBytes"] = str(int(state.get("tcp_d_out", 0.0))) + out[1].tags["data.retransmittedSegs"] = str(int(state.get("tcp_d_rs", 0.0))) + out[1].tags["data.sendWindowSize"] = str(int(state.get("tcp_swnd", 0.0))) + out[1].tags["data.congestionWindowSize"] = str(int(state.get("tcp_cwnd", 0.0))) + out[1].tags["data.receiveWindowSize"] = str(int(state.get("tcp_rwnd", 0.0))) + out[1].tags["data.roundTripTime"] = str(int(state.get("tcp_rtt", 0.0))) + out[1].tags["data.connections"] = str(int(state.get("tcp_conn", 0.0))) + state["tcp_d_in"] = 0.0 + state["tcp_d_out"] = 0.0 + state["tcp_d_rs"] = 0.0 + state["tcp_acc"] = state.get("tcp_acc", 0.0) + float(d_in + d_out) + state["tcp_d_in"] = state.get("tcp_d_in", 0.0) + float(d_in) + state["tcp_d_out"] = state.get("tcp_d_out", 0.0) + float(d_out) + state["tcp_d_rs"] = state.get("tcp_d_rs", 0.0) + float(d_rs) + state["tcp_swnd"] = swnd + state["tcp_cwnd"] = cwnd + state["tcp_rwnd"] = rwnd + state["tcp_rtt"] = rtt + state["tcp_conn"] = conn return out return [metric] From 38656803fd81672e9c8ab389fe60d891ddea5314 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Thu, 29 Oct 2026 18:43:39 +0530 Subject: [PATCH 15/17] DLPX-100030 telegraf.processors.fluentd: drop usage_ prefix from CPU data.* fields Rename data.usage_* -> data.* on the cpu compat metric so Splunk reads e.g. data.idle, data.user, data.system (matching Java's naming convention) instead of data.usage_idle. State keys (cpu_d_usr etc.) are unchanged. Verified on ev-dev and dbs-test: processor loads cleanly, HEC fields now carry data.user / data.system / data.idle / data.iowait / data.irq / data.softirq / data.steal. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/telegraf.processors.fluentd | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/telegraf/telegraf.processors.fluentd b/telegraf/telegraf.processors.fluentd index dd6a2d2..69a7c15 100644 --- a/telegraf/telegraf.processors.fluentd +++ b/telegraf/telegraf.processors.fluentd @@ -80,13 +80,13 @@ def apply(metric): out = [metric] out = out + flush_gauge("cpu", "system.cpu.util", "pct", bucket) if len(out) > 1: - out[1].tags["data.usage_user"] = str(state.get("cpu_d_usr", 0.0)) - out[1].tags["data.usage_system"] = str(state.get("cpu_d_sys", 0.0)) - out[1].tags["data.usage_idle"] = str(state.get("cpu_d_idl", 0.0)) - out[1].tags["data.usage_iowait"] = str(state.get("cpu_d_iow", 0.0)) - out[1].tags["data.usage_irq"] = str(state.get("cpu_d_irq", 0.0)) - out[1].tags["data.usage_softirq"] = str(state.get("cpu_d_sir", 0.0)) - out[1].tags["data.usage_steal"] = str(state.get("cpu_d_stl", 0.0)) + out[1].tags["data.user"] = str(state.get("cpu_d_usr", 0.0)) + out[1].tags["data.system"] = str(state.get("cpu_d_sys", 0.0)) + out[1].tags["data.idle"] = str(state.get("cpu_d_idl", 0.0)) + out[1].tags["data.iowait"] = str(state.get("cpu_d_iow", 0.0)) + out[1].tags["data.irq"] = str(state.get("cpu_d_irq", 0.0)) + out[1].tags["data.softirq"] = str(state.get("cpu_d_sir", 0.0)) + out[1].tags["data.steal"] = str(state.get("cpu_d_stl", 0.0)) state["cpu_last"] = 100.0 - float(metric.fields["usage_idle"]) state["cpu_d_usr"] = float(metric.fields.get("usage_user", 0)) state["cpu_d_sys"] = float(metric.fields.get("usage_system", 0)) From 859cdefe9929280f100a811d668ae15c06563299 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Thu, 29 Oct 2026 19:43:38 +0530 Subject: [PATCH 16/17] DLPX-100030 telegraf: order processor after storage_io, drop TCP last-write-wins, guard envsubst Three focused fixes from PR #135 review: 1. Add order = 4 on [[processors.starlark]] so our processor runs AFTER the base-config storage_io processors (orders 1-3). Without this, the storage name=total filter may race with our estat accumulation, causing read + write + total rows all to be summed into nfs/iscsi counts (double-count). 2. Drop 5 TCP data.* fields that were last-write-wins across endpoints (sendWindowSize, congestionWindowSize, receiveWindowSize, roundTripTime, connections). Values reflected whichever connection happened to be the last one processed in the bucket, not any meaningful aggregate. The remaining 3 fields (data.inBytes, data.outBytes, data.retransmittedSegs) are true per-bucket deltas via cumulative_delta. 3. Validate DELPHIX_SYSTEM_UUID / DELPHIX_SYSTEM_VERSION against a safe char set ([A-Za-z0-9._+-]+) before envsubst inlines them into the Starlark triple-quoted TOML string. Unsafe values fall back to "unknown" with a syslog warning, so the service still starts (metrics tag identity as unknown rather than Telegraf failing to parse the config). Verified end-to-end on ev-dev.dlpxdc.co and dbs-test.dlpxdc.co: - Both engines load cleanly - All 6 compat metrics still emitting (system.cpu.util.pct through system.tcp.total.bytes) - TCP now has 3 data.* fields (previously 8); other metrics unchanged - Identity tags still present with real UUID/version values Co-Authored-By: Claude Sonnet 4.6 (1M context) --- telegraf/delphix-telegraf-service | 12 +++++++++++ telegraf/telegraf.processors.fluentd | 32 +++++++++++----------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/telegraf/delphix-telegraf-service b/telegraf/delphix-telegraf-service index 32bc2fa..186b493 100755 --- a/telegraf/delphix-telegraf-service +++ b/telegraf/delphix-telegraf-service @@ -16,6 +16,18 @@ TELEGRAF_CONFIG_DIR=/etc/telegraf/telegraf.d export DELPHIX_SYSTEM_UUID=$(get-system-uuid 2>/dev/null || echo "unknown") export DELPHIX_SYSTEM_VERSION=$(get-appliance-version 2>/dev/null || echo "unknown") +# Guard against envsubst-injection into the Starlark triple-quoted TOML string: +# allow only characters that can safely appear in a Starlark double-quoted +# string literal. Anything else -> fall back to "unknown" so the service still +# starts (metrics will tag identity as unknown rather than failing to launch). +if [[ ! "$DELPHIX_SYSTEM_UUID" =~ ^[A-Za-z0-9._+-]+$ ]]; then + logger -t delphix-telegraf "WARNING: get-system-uuid returned a value with unsafe characters — using 'unknown' as fallback" + export DELPHIX_SYSTEM_UUID="unknown" +fi +if [[ ! "$DELPHIX_SYSTEM_VERSION" =~ ^[A-Za-z0-9._+-]+$ ]]; then + logger -t delphix-telegraf "WARNING: get-appliance-version returned a value with unsafe characters — using 'unknown' as fallback" + export DELPHIX_SYSTEM_VERSION="unknown" +fi function engine_is_dct() { /usr/bin/get-appliance-variant | grep -q "dct" >/dev/null diff --git a/telegraf/telegraf.processors.fluentd b/telegraf/telegraf.processors.fluentd index 69a7c15..25ab3a6 100644 --- a/telegraf/telegraf.processors.fluentd +++ b/telegraf/telegraf.processors.fluentd @@ -11,6 +11,10 @@ # Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists. [[processors.starlark]] namepass = ["cpu", "diskio", "net", "estat_nfs", "estat_iscsi", "tcp_stats"] + # Run after the base-config storage_io processors (order 1-3): the name=total + # row filter must execute first, otherwise read + write + total rows all land + # in our accumulators and NFS/iSCSI ops counts come out doubled. + order = 4 source = ''' UUID = "${DELPHIX_SYSTEM_UUID}" VERSION = "${DELPHIX_SYSTEM_VERSION}" @@ -225,31 +229,26 @@ def apply(metric): state["dsk_d_wt"] = state.get("dsk_d_wt", 0.0) + float(d_wt) return out - # system.tcp.total.bytes — data.* includes bytes + retransmits + windows + rtt. + # system.tcp.total.bytes — data.* carries true per-bucket deltas only. + # swnd/cwnd/rwnd/rtt/connections were dropped: they were last-write-wins + # across endpoints, so the emitted value reflected whichever connection + # happened to be processed last in the bucket rather than any meaningful + # aggregate. Per-endpoint window/rtt would require per-endpoint events, + # which is out of scope for the aggregated system.tcp.total.bytes metric. if name == "tcp_stats": key = metric.tags.get("laddr", "") + "|" + metric.tags.get("raddr", "") + "|" + metric.tags.get("service", "") inb = int(metric.fields.get("inbytes", 0)) outb = int(metric.fields.get("outbytes", 0)) rseg = int(metric.fields.get("retranssegs", 0)) - swnd = int(metric.fields.get("swnd", 0)) - cwnd = int(metric.fields.get("cwnd", 0)) - rwnd = int(metric.fields.get("rwnd", 0)) - rtt = int(metric.fields.get("rtt", 0)) - conn = int(metric.fields.get("connections", 0)) d_in = cumulative_delta("tcp_in_" + key, inb) d_out = cumulative_delta("tcp_out_" + key, outb) d_rs = cumulative_delta("tcp_rs_" + key, rseg) out = [metric] out = out + flush_counter("tcp", "system.tcp.total", "bytes", bucket) if len(out) > 1: - out[1].tags["data.inBytes"] = str(int(state.get("tcp_d_in", 0.0))) - out[1].tags["data.outBytes"] = str(int(state.get("tcp_d_out", 0.0))) - out[1].tags["data.retransmittedSegs"] = str(int(state.get("tcp_d_rs", 0.0))) - out[1].tags["data.sendWindowSize"] = str(int(state.get("tcp_swnd", 0.0))) - out[1].tags["data.congestionWindowSize"] = str(int(state.get("tcp_cwnd", 0.0))) - out[1].tags["data.receiveWindowSize"] = str(int(state.get("tcp_rwnd", 0.0))) - out[1].tags["data.roundTripTime"] = str(int(state.get("tcp_rtt", 0.0))) - out[1].tags["data.connections"] = str(int(state.get("tcp_conn", 0.0))) + out[1].tags["data.inBytes"] = str(int(state.get("tcp_d_in", 0.0))) + out[1].tags["data.outBytes"] = str(int(state.get("tcp_d_out", 0.0))) + out[1].tags["data.retransmittedSegs"] = str(int(state.get("tcp_d_rs", 0.0))) state["tcp_d_in"] = 0.0 state["tcp_d_out"] = 0.0 state["tcp_d_rs"] = 0.0 @@ -257,11 +256,6 @@ def apply(metric): state["tcp_d_in"] = state.get("tcp_d_in", 0.0) + float(d_in) state["tcp_d_out"] = state.get("tcp_d_out", 0.0) + float(d_out) state["tcp_d_rs"] = state.get("tcp_d_rs", 0.0) + float(d_rs) - state["tcp_swnd"] = swnd - state["tcp_cwnd"] = cwnd - state["tcp_rwnd"] = rwnd - state["tcp_rtt"] = rtt - state["tcp_conn"] = conn return out return [metric] From bc4783fc0bc81c39170485c3307b5a409a7e13d5 Mon Sep 17 00:00:00 2001 From: dbshah12 Date: Fri, 30 Oct 2026 13:54:48 +0530 Subject: [PATCH 17/17] DLPX-100030 telegraf: add fluentd HTTP output for 3rd-party metric forwarding Co-Authored-By: Claude Opus 4.7 (1M context) --- debian/rules | 2 +- telegraf/delphix-telegraf-service | 53 +++++- telegraf/perf_fluentd | 64 +++++++ telegraf/telegraf.outputs.fluentd | 22 +++ telegraf/telegraf.processors.fluentd | 262 +++++++++++++++++++++++++++ 5 files changed, 400 insertions(+), 3 deletions(-) create mode 100644 telegraf/perf_fluentd create mode 100644 telegraf/telegraf.outputs.fluentd create mode 100644 telegraf/telegraf.processors.fluentd diff --git a/debian/rules b/debian/rules index 00b38aa..d573e6c 100755 --- a/debian/rules +++ b/debian/rules @@ -23,7 +23,7 @@ override_dh_auto_install: dh_install build/cmd/* /usr/bin dh_install lib/* /usr/share/performance-diagnostics/lib dh_install bpf/* /usr/share/performance-diagnostics/bpf - dh_install telegraf/delphix-telegraf-service telegraf/perf_playbook /usr/bin + dh_install telegraf/delphix-telegraf-service telegraf/perf_playbook telegraf/perf_fluentd /usr/bin dh_install telegraf/delphix-telegraf.service /lib/systemd/system dh_install telegraf/telegraf* telegraf/*.sh /etc/telegraf dh_install influxdb/delphix-influxdb-service influxdb/delphix-influxdb-init influxdb/perf_influxdb /usr/bin diff --git a/telegraf/delphix-telegraf-service b/telegraf/delphix-telegraf-service index 3c9a4dc..186b493 100755 --- a/telegraf/delphix-telegraf-service +++ b/telegraf/delphix-telegraf-service @@ -5,12 +5,30 @@ DCT_INPUTS=/etc/telegraf/telegraf.inputs.dct PLAYBOOK_INPUTS=/etc/telegraf/telegraf.inputs.playbook STORAGE_IO_INPUTS=/etc/telegraf/telegraf.inputs.storage_io INFLUXDB_OUTPUT=/etc/telegraf/telegraf.outputs.influxdb +FLUENTD_OUTPUT=/etc/telegraf/telegraf.outputs.fluentd +FLUENTD_PROCESSOR=/etc/telegraf/telegraf.processors.fluentd PLAYBOOK_FLAG=/etc/telegraf/PLAYBOOK_ENABLED INFLUXDB_FLAG=/etc/telegraf/INFLUXDB_ENABLED +FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED TELEGRAF_CONFIG=/etc/telegraf/telegraf.conf TELEGRAF_CONFIG_DIR=/etc/telegraf/telegraf.d +export DELPHIX_SYSTEM_UUID=$(get-system-uuid 2>/dev/null || echo "unknown") +export DELPHIX_SYSTEM_VERSION=$(get-appliance-version 2>/dev/null || echo "unknown") +# Guard against envsubst-injection into the Starlark triple-quoted TOML string: +# allow only characters that can safely appear in a Starlark double-quoted +# string literal. Anything else -> fall back to "unknown" so the service still +# starts (metrics will tag identity as unknown rather than failing to launch). +if [[ ! "$DELPHIX_SYSTEM_UUID" =~ ^[A-Za-z0-9._+-]+$ ]]; then + logger -t delphix-telegraf "WARNING: get-system-uuid returned a value with unsafe characters — using 'unknown' as fallback" + export DELPHIX_SYSTEM_UUID="unknown" +fi +if [[ ! "$DELPHIX_SYSTEM_VERSION" =~ ^[A-Za-z0-9._+-]+$ ]]; then + logger -t delphix-telegraf "WARNING: get-appliance-version returned a value with unsafe characters — using 'unknown' as fallback" + export DELPHIX_SYSTEM_VERSION="unknown" +fi + function engine_is_dct() { /usr/bin/get-appliance-variant | grep -q "dct" >/dev/null [[ "$?" == "0" ]] @@ -29,6 +47,10 @@ function influxdb_is_enabled() { [[ -f $INFLUXDB_FLAG ]] } +function fluentd_is_enabled() { + [[ -f $FLUENTD_FLAG ]] +} + rm -f $TELEGRAF_CONFIG if engine_is_object_based; then @@ -51,16 +73,43 @@ else fi fi -if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then +has_output=false + +# Include storage I/O inputs (estat_nfs, estat_iscsi, estat_backend-io) whenever +# any active output consumes those measurements — either InfluxDB or fluentd. +if { influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; } || + { fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; }; then if [[ -f $STORAGE_IO_INPUTS ]]; then cat $STORAGE_IO_INPUTS >> $TELEGRAF_CONFIG fi +fi + +if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then cat $INFLUXDB_OUTPUT >> $TELEGRAF_CONFIG + has_output=true else if influxdb_is_enabled; then logger -t delphix-telegraf "WARNING: INFLUXDB_ENABLED is set but $INFLUXDB_OUTPUT is missing — metrics will be discarded. Run delphix-influxdb-init to restore InfluxDB output." fi - # No InfluxDB output configured. Add discard so Telegraf can start — +fi + +if fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; then + if [[ ! -f $FLUENTD_PROCESSOR ]]; then + logger -t delphix-telegraf "ERROR: FLUENTD_ENABLED is set but $FLUENTD_PROCESSOR is missing — raw Telegraf metric names will be forwarded, breaking existing Splunk dashboards. Reinstall the delphix-perfdiag package to restore it." + else + # Interpolate DELPHIX_SYSTEM_UUID / DELPHIX_SYSTEM_VERSION into the + # processor's starlark source. Telegraf's own ${VAR} substitution does + # not reach inside triple-quoted TOML strings, so we do it here. + envsubst '${DELPHIX_SYSTEM_UUID} ${DELPHIX_SYSTEM_VERSION}' < $FLUENTD_PROCESSOR >> $TELEGRAF_CONFIG + fi + cat $FLUENTD_OUTPUT >> $TELEGRAF_CONFIG + has_output=true +elif fluentd_is_enabled; then + logger -t delphix-telegraf "WARNING: FLUENTD_ENABLED is set but $FLUENTD_OUTPUT is missing — metrics will be discarded. Reinstall the delphix-perfdiag package to restore fluentd output." +fi + +if [[ "$has_output" != "true" ]]; then + # No output configured. Add discard so Telegraf can start — # it requires at least one output plugin. echo "[[outputs.discard]]" >> $TELEGRAF_CONFIG fi diff --git a/telegraf/perf_fluentd b/telegraf/perf_fluentd new file mode 100644 index 0000000..6ad851b --- /dev/null +++ b/telegraf/perf_fluentd @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright (c) 2026 by Delphix. All rights reserved. +# +# Script that enables and disables fluentd 3rd-party metric forwarding for Telegraf. +# + +FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED + +function die() { + echo -e "$(date +%T:%N:%z): $(basename $0): $*" >&2 + exit 1 +} + +[[ $EUID -ne 0 ]] && die "must be run as root" + +function usage() { + echo "$(basename $0): $*" >&2 + echo "Usage: $(basename $0) [enable|disable|status]" + exit 2 +} + +function enable_fluentd() { + date + if [[ -f $FLUENTD_FLAG ]]; then + echo "fluentd forwarding already enabled — nothing to do" + return 0 + fi + echo "Enabling fluentd 3rd-party metric forwarding" + touch $FLUENTD_FLAG || die "Failed to create $FLUENTD_FLAG" + systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf" +} + +function disable_fluentd() { + date + if [[ ! -f $FLUENTD_FLAG ]]; then + echo "fluentd forwarding already disabled — nothing to do" + return 0 + fi + echo "Disabling fluentd 3rd-party metric forwarding" + rm -f $FLUENTD_FLAG + systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf" +} + +function status_fluentd() { + if [[ -f $FLUENTD_FLAG ]]; then + echo "fluentd forwarding: enabled" + else + echo "fluentd forwarding: disabled" + fi + echo -n "delphix-telegraf: " + systemctl is-active delphix-telegraf +} + +if [[ $# -ne 1 ]]; then + usage +fi + +case "$1" in +enable) enable_fluentd ;; +disable) disable_fluentd ;; +status) status_fluentd ;; +*) usage ;; +esac diff --git a/telegraf/telegraf.outputs.fluentd b/telegraf/telegraf.outputs.fluentd new file mode 100644 index 0000000..061462b --- /dev/null +++ b/telegraf/telegraf.outputs.fluentd @@ -0,0 +1,22 @@ +# Telegraf HTTP output → local fluentd HTTP source. +# Forwards performance analytics metrics to fluentd, which routes them to +# third-party monitoring systems (Splunk, Datadog) via existing +# rules in fluent.conf.stg. +# Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists. +[[outputs.http]] + url = "http://localhost:9880/delphix.metrics.platform" + method = "POST" + data_format = "splunkmetric" + splunkmetric_hec_routing = true + # One POST per metric: batched splunkmetric emits concatenated JSON with no delimiter, which fluentd's JSON parser silently truncates to the first object. + use_batch_format = false + flush_interval = "60s" + timeout = "10s" + # Buffer a few minutes of compat metrics so a brief fluentd restart doesn't + # lose data. The processor emits ~6 metrics per flush (one per Java metric + # name), so 2048 buffers > 5 hours of history — well above typical downtime. + metric_buffer_limit = 2048 + namepass = ["system.cpu.util", "system.net.total", "system.disk.ops", + "system.nfs.ops", "system.iscsi.ops", "system.tcp.total"] + [outputs.http.headers] + Content-Type = "application/json" diff --git a/telegraf/telegraf.processors.fluentd b/telegraf/telegraf.processors.fluentd new file mode 100644 index 0000000..25ab3a6 --- /dev/null +++ b/telegraf/telegraf.processors.fluentd @@ -0,0 +1,262 @@ +# Starlark processor — maps Telegraf measurements to Java InsightLogger metric +# names (system.*) with semantics matching Java's pushInsightMetric. +# Attaches rich data.* breakdown fields alongside _value (Telegraf-native names). +# +# Note: round() triggers a Telegraf 1.40.2 starlark resolver bug ("undefined: +# state"). We emit numeric values via plain str() instead. +# +# ${DELPHIX_SYSTEM_UUID} / ${DELPHIX_SYSTEM_VERSION} are substituted at TOML +# parse time from env vars exported by delphix-telegraf-service. +# +# Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists. +[[processors.starlark]] + namepass = ["cpu", "diskio", "net", "estat_nfs", "estat_iscsi", "tcp_stats"] + # Run after the base-config storage_io processors (order 1-3): the name=total + # row filter must execute first, otherwise read + write + total rows all land + # in our accumulators and NFS/iSCSI ops counts come out doubled. + order = 4 + source = ''' +UUID = "${DELPHIX_SYSTEM_UUID}" +VERSION = "${DELPHIX_SYSTEM_VERSION}" +BUCKET_NS = 60 * 1000 * 1000 * 1000 + +def tag_identity(m): + m.tags["systemUniqueId"] = UUID + m.tags["systemVersion"] = VERSION + return m + +def cumulative_delta(key, current): + # Delta since the previous call for this key, or 0 on the first call + # (no previous baseline). Guards against counter wrap / device reset. + prev = state.get(key, None) + state[key] = current + if prev == None: + return 0 + d = current - prev + if d < 0: + return 0 + return d + +def flush_counter(prefix, measurement, field, bucket): + # If we have crossed a 60s bucket boundary, emit the previous bucket's + # accumulated sum and reset the accumulator for the new bucket. Returns a + # list (possibly empty) of metrics to emit. + bk = prefix + "_bucket" + ak = prefix + "_acc" + prev_bucket = state.get(bk, None) + out = [] + if prev_bucket != None and prev_bucket != bucket: + acc = state.get(ak, 0.0) + m = Metric(measurement) + m.time = prev_bucket * BUCKET_NS + m.fields[field] = acc + out.append(tag_identity(m)) + state[ak] = 0.0 + state[bk] = bucket + return out + +def flush_gauge(prefix, measurement, field, bucket): + # Gauge version: emit the previous bucket's last-seen value when the + # bucket boundary is crossed. "last" is updated after this call. + bk = prefix + "_bucket" + lk = prefix + "_last" + prev_bucket = state.get(bk, None) + out = [] + if prev_bucket != None and prev_bucket != bucket and lk in state: + m = Metric(measurement) + m.time = prev_bucket * BUCKET_NS + m.fields[field] = state[lk] + out.append(tag_identity(m)) + state[bk] = bucket + return out + +def apply(metric): + name = metric.name + ts = metric.time + bucket = ts // BUCKET_NS + + # system.cpu.util.pct — data.* carries all 7 cpu-time breakdown fields. + if name == "cpu": + if metric.tags.get("cpu") != "cpu-total": + return [metric] + if "usage_idle" not in metric.fields: + return [metric] + out = [metric] + out = out + flush_gauge("cpu", "system.cpu.util", "pct", bucket) + if len(out) > 1: + out[1].tags["data.user"] = str(state.get("cpu_d_usr", 0.0)) + out[1].tags["data.system"] = str(state.get("cpu_d_sys", 0.0)) + out[1].tags["data.idle"] = str(state.get("cpu_d_idl", 0.0)) + out[1].tags["data.iowait"] = str(state.get("cpu_d_iow", 0.0)) + out[1].tags["data.irq"] = str(state.get("cpu_d_irq", 0.0)) + out[1].tags["data.softirq"] = str(state.get("cpu_d_sir", 0.0)) + out[1].tags["data.steal"] = str(state.get("cpu_d_stl", 0.0)) + state["cpu_last"] = 100.0 - float(metric.fields["usage_idle"]) + state["cpu_d_usr"] = float(metric.fields.get("usage_user", 0)) + state["cpu_d_sys"] = float(metric.fields.get("usage_system", 0)) + state["cpu_d_idl"] = float(metric.fields.get("usage_idle", 0)) + state["cpu_d_iow"] = float(metric.fields.get("usage_iowait", 0)) + state["cpu_d_irq"] = float(metric.fields.get("usage_irq", 0)) + state["cpu_d_sir"] = float(metric.fields.get("usage_softirq", 0)) + state["cpu_d_stl"] = float(metric.fields.get("usage_steal", 0)) + return out + + # system.nfs.ops.count — data.* includes read/write ops + latency + throughput. + if name == "estat_nfs": + if "iops(/s)" not in metric.fields: + return [metric] + iops = float(metric.fields["iops(/s)"]) * 10.0 + latency = float(metric.fields.get("avg latency(us)", 0)) + throughput = float(metric.fields.get("throughput(k/s)", 0)) + direction = metric.tags.get("name", "") + out = [metric] + out = out + flush_counter("nfs", "system.nfs.ops", "count", bucket) + if len(out) > 1: + out[1].tags["data.readOps"] = str(int(state.get("nfs_d_r", 0.0))) + out[1].tags["data.writeOps"] = str(int(state.get("nfs_d_w", 0.0))) + out[1].tags["data.readLatency"] = str(state.get("nfs_rl", 0.0)) + out[1].tags["data.writeLatency"] = str(state.get("nfs_wl", 0.0)) + out[1].tags["data.readThroughput"] = str(state.get("nfs_rt", 0.0)) + out[1].tags["data.writeThroughput"] = str(state.get("nfs_wt", 0.0)) + state["nfs_d_r"] = 0.0 + state["nfs_d_w"] = 0.0 + state["nfs_acc"] = state.get("nfs_acc", 0.0) + iops + if direction == "read": + state["nfs_d_r"] = state.get("nfs_d_r", 0.0) + iops + state["nfs_rl"] = latency + state["nfs_rt"] = throughput + else: + state["nfs_d_w"] = state.get("nfs_d_w", 0.0) + iops + state["nfs_wl"] = latency + state["nfs_wt"] = throughput + return out + + # system.iscsi.ops.count — same shape as nfs. + if name == "estat_iscsi": + if "iops(/s)" not in metric.fields: + return [metric] + iops = float(metric.fields["iops(/s)"]) * 10.0 + latency = float(metric.fields.get("avg latency(us)", 0)) + throughput = float(metric.fields.get("throughput(k/s)", 0)) + direction = metric.tags.get("name", "") + out = [metric] + out = out + flush_counter("iscsi", "system.iscsi.ops", "count", bucket) + if len(out) > 1: + out[1].tags["data.readOps"] = str(int(state.get("isc_d_r", 0.0))) + out[1].tags["data.writeOps"] = str(int(state.get("isc_d_w", 0.0))) + out[1].tags["data.readLatency"] = str(state.get("isc_rl", 0.0)) + out[1].tags["data.writeLatency"] = str(state.get("isc_wl", 0.0)) + out[1].tags["data.readThroughput"] = str(state.get("isc_rt", 0.0)) + out[1].tags["data.writeThroughput"] = str(state.get("isc_wt", 0.0)) + state["isc_d_r"] = 0.0 + state["isc_d_w"] = 0.0 + state["iscsi_acc"] = state.get("iscsi_acc", 0.0) + iops + if direction == "read": + state["isc_d_r"] = state.get("isc_d_r", 0.0) + iops + state["isc_rl"] = latency + state["isc_rt"] = throughput + else: + state["isc_d_w"] = state.get("isc_d_w", 0.0) + iops + state["isc_wl"] = latency + state["isc_wt"] = throughput + return out + + # system.net.total.bytes — data.* includes bytes + packets. + if name == "net": + iface = metric.tags.get("interface", "") + recv = int(metric.fields.get("bytes_recv", 0)) + sent = int(metric.fields.get("bytes_sent", 0)) + recv_pkt = int(metric.fields.get("packets_recv", 0)) + sent_pkt = int(metric.fields.get("packets_sent", 0)) + d_in = cumulative_delta("net_r_" + iface, recv) + d_out = cumulative_delta("net_s_" + iface, sent) + d_ip = cumulative_delta("net_rp_" + iface, recv_pkt) + d_op = cumulative_delta("net_sp_" + iface, sent_pkt) + out = [metric] + out = out + flush_counter("net", "system.net.total", "bytes", bucket) + if len(out) > 1: + out[1].tags["data.inBytes"] = str(int(state.get("net_d_in", 0.0))) + out[1].tags["data.outBytes"] = str(int(state.get("net_d_out", 0.0))) + out[1].tags["data.inPackets"] = str(int(state.get("net_d_ip", 0.0))) + out[1].tags["data.outPackets"] = str(int(state.get("net_d_op", 0.0))) + state["net_d_in"] = 0.0 + state["net_d_out"] = 0.0 + state["net_d_ip"] = 0.0 + state["net_d_op"] = 0.0 + state["net_acc"] = state.get("net_acc", 0.0) + float(d_in + d_out) + state["net_d_in"] = state.get("net_d_in", 0.0) + float(d_in) + state["net_d_out"] = state.get("net_d_out", 0.0) + float(d_out) + state["net_d_ip"] = state.get("net_d_ip", 0.0) + float(d_ip) + state["net_d_op"] = state.get("net_d_op", 0.0) + float(d_op) + return out + + # system.disk.ops.count — data.* includes reads/writes + bytes + times. + if name == "diskio": + dev = metric.tags.get("name", "") + r = int(metric.fields.get("reads", 0)) + w = int(metric.fields.get("writes", 0)) + rb = int(metric.fields.get("read_bytes", 0)) + wb = int(metric.fields.get("write_bytes", 0)) + rt = int(metric.fields.get("read_time", 0)) + wt = int(metric.fields.get("write_time", 0)) + d_r = cumulative_delta("disk_r_" + dev, r) + d_w = cumulative_delta("disk_w_" + dev, w) + d_rb = cumulative_delta("disk_rb_" + dev, rb) + d_wb = cumulative_delta("disk_wb_" + dev, wb) + d_rt = cumulative_delta("disk_rt_" + dev, rt) + d_wt = cumulative_delta("disk_wt_" + dev, wt) + out = [metric] + out = out + flush_counter("disk", "system.disk.ops", "count", bucket) + if len(out) > 1: + out[1].tags["data.reads"] = str(int(state.get("dsk_d_r", 0.0))) + out[1].tags["data.writes"] = str(int(state.get("dsk_d_w", 0.0))) + out[1].tags["data.bytesRead"] = str(int(state.get("dsk_d_rb", 0.0))) + out[1].tags["data.bytesWritten"] = str(int(state.get("dsk_d_wb", 0.0))) + out[1].tags["data.readTime"] = str(int(state.get("dsk_d_rt", 0.0))) + out[1].tags["data.writeTime"] = str(int(state.get("dsk_d_wt", 0.0))) + state["dsk_d_r"] = 0.0 + state["dsk_d_w"] = 0.0 + state["dsk_d_rb"] = 0.0 + state["dsk_d_wb"] = 0.0 + state["dsk_d_rt"] = 0.0 + state["dsk_d_wt"] = 0.0 + state["disk_acc"] = state.get("disk_acc", 0.0) + float(d_r + d_w) + state["dsk_d_r"] = state.get("dsk_d_r", 0.0) + float(d_r) + state["dsk_d_w"] = state.get("dsk_d_w", 0.0) + float(d_w) + state["dsk_d_rb"] = state.get("dsk_d_rb", 0.0) + float(d_rb) + state["dsk_d_wb"] = state.get("dsk_d_wb", 0.0) + float(d_wb) + state["dsk_d_rt"] = state.get("dsk_d_rt", 0.0) + float(d_rt) + state["dsk_d_wt"] = state.get("dsk_d_wt", 0.0) + float(d_wt) + return out + + # system.tcp.total.bytes — data.* carries true per-bucket deltas only. + # swnd/cwnd/rwnd/rtt/connections were dropped: they were last-write-wins + # across endpoints, so the emitted value reflected whichever connection + # happened to be processed last in the bucket rather than any meaningful + # aggregate. Per-endpoint window/rtt would require per-endpoint events, + # which is out of scope for the aggregated system.tcp.total.bytes metric. + if name == "tcp_stats": + key = metric.tags.get("laddr", "") + "|" + metric.tags.get("raddr", "") + "|" + metric.tags.get("service", "") + inb = int(metric.fields.get("inbytes", 0)) + outb = int(metric.fields.get("outbytes", 0)) + rseg = int(metric.fields.get("retranssegs", 0)) + d_in = cumulative_delta("tcp_in_" + key, inb) + d_out = cumulative_delta("tcp_out_" + key, outb) + d_rs = cumulative_delta("tcp_rs_" + key, rseg) + out = [metric] + out = out + flush_counter("tcp", "system.tcp.total", "bytes", bucket) + if len(out) > 1: + out[1].tags["data.inBytes"] = str(int(state.get("tcp_d_in", 0.0))) + out[1].tags["data.outBytes"] = str(int(state.get("tcp_d_out", 0.0))) + out[1].tags["data.retransmittedSegs"] = str(int(state.get("tcp_d_rs", 0.0))) + state["tcp_d_in"] = 0.0 + state["tcp_d_out"] = 0.0 + state["tcp_d_rs"] = 0.0 + state["tcp_acc"] = state.get("tcp_acc", 0.0) + float(d_in + d_out) + state["tcp_d_in"] = state.get("tcp_d_in", 0.0) + float(d_in) + state["tcp_d_out"] = state.get("tcp_d_out", 0.0) + float(d_out) + state["tcp_d_rs"] = state.get("tcp_d_rs", 0.0) + float(d_rs) + return out + + return [metric] +'''