DLPX-100030 telegraf: add fluentd HTTP output for 3rd-party metric forwarding

debian/rules

@@ -23,7 +23,7 @@ override_dh_auto_install:
 	dh_install build/cmd/* /usr/bin
 	dh_install lib/* /usr/share/performance-diagnostics/lib
 	dh_install bpf/* /usr/share/performance-diagnostics/bpf
-	dh_install telegraf/delphix-telegraf-service telegraf/perf_playbook /usr/bin
+	dh_install telegraf/delphix-telegraf-service telegraf/perf_playbook telegraf/perf_fluentd /usr/bin
 	dh_install telegraf/delphix-telegraf.service /lib/systemd/system
 	dh_install telegraf/telegraf* telegraf/*.sh /etc/telegraf
 	dh_install influxdb/delphix-influxdb-service influxdb/delphix-influxdb-init influxdb/perf_influxdb /usr/bin

telegraf/delphix-telegraf-service

@@ -5,12 +5,30 @@ DCT_INPUTS=/etc/telegraf/telegraf.inputs.dct
 PLAYBOOK_INPUTS=/etc/telegraf/telegraf.inputs.playbook
 STORAGE_IO_INPUTS=/etc/telegraf/telegraf.inputs.storage_io
 INFLUXDB_OUTPUT=/etc/telegraf/telegraf.outputs.influxdb
+FLUENTD_OUTPUT=/etc/telegraf/telegraf.outputs.fluentd
+FLUENTD_PROCESSOR=/etc/telegraf/telegraf.processors.fluentd
 PLAYBOOK_FLAG=/etc/telegraf/PLAYBOOK_ENABLED
 INFLUXDB_FLAG=/etc/telegraf/INFLUXDB_ENABLED
+FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED
 TELEGRAF_CONFIG=/etc/telegraf/telegraf.conf
 TELEGRAF_CONFIG_DIR=/etc/telegraf/telegraf.d
 
 
+export DELPHIX_SYSTEM_UUID=$(get-system-uuid 2>/dev/null || echo "unknown")
+export DELPHIX_SYSTEM_VERSION=$(get-appliance-version 2>/dev/null || echo "unknown")
+# Guard against envsubst-injection into the Starlark triple-quoted TOML string:
+# allow only characters that can safely appear in a Starlark double-quoted
+# string literal. Anything else -> fall back to "unknown" so the service still
+# starts (metrics will tag identity as unknown rather than failing to launch).
+if [[ ! "$DELPHIX_SYSTEM_UUID" =~ ^[A-Za-z0-9._+-]+$ ]]; then
+	logger -t delphix-telegraf "WARNING: get-system-uuid returned a value with unsafe characters — using 'unknown' as fallback"
+	export DELPHIX_SYSTEM_UUID="unknown"
+fi
+if [[ ! "$DELPHIX_SYSTEM_VERSION" =~ ^[A-Za-z0-9._+-]+$ ]]; then
+	logger -t delphix-telegraf "WARNING: get-appliance-version returned a value with unsafe characters — using 'unknown' as fallback"
+	export DELPHIX_SYSTEM_VERSION="unknown"
+fi
+
 function engine_is_dct() {
 	/usr/bin/get-appliance-variant | grep -q "dct" >/dev/null
 	[[ "$?" == "0" ]]
@@ -29,6 +47,10 @@ function influxdb_is_enabled() {
 	[[ -f $INFLUXDB_FLAG ]]
 }
 
+function fluentd_is_enabled() {
+	[[ -f $FLUENTD_FLAG ]]
+}
+
 rm -f $TELEGRAF_CONFIG
 
 if engine_is_object_based; then
@@ -51,16 +73,43 @@ else
 	fi
 fi
 
-if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then
+has_output=false
+
+# Include storage I/O inputs (estat_nfs, estat_iscsi, estat_backend-io) whenever
+# any active output consumes those measurements — either InfluxDB or fluentd.
+if { influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; } ||
+   { fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; }; then
 	if [[ -f $STORAGE_IO_INPUTS ]]; then
 		cat $STORAGE_IO_INPUTS >> $TELEGRAF_CONFIG
 	fi
+fi
+
+if influxdb_is_enabled && [[ -f $INFLUXDB_OUTPUT ]]; then
 	cat $INFLUXDB_OUTPUT >> $TELEGRAF_CONFIG
+	has_output=true
 else
 	if influxdb_is_enabled; then
 		logger -t delphix-telegraf "WARNING: INFLUXDB_ENABLED is set but $INFLUXDB_OUTPUT is missing — metrics will be discarded. Run delphix-influxdb-init to restore InfluxDB output."
 	fi
-	# No InfluxDB output configured. Add discard so Telegraf can start —
+fi
+
+if fluentd_is_enabled && [[ -f $FLUENTD_OUTPUT ]]; then
+	if [[ ! -f $FLUENTD_PROCESSOR ]]; then
+		logger -t delphix-telegraf "ERROR: FLUENTD_ENABLED is set but $FLUENTD_PROCESSOR is missing — raw Telegraf metric names will be forwarded, breaking existing Splunk dashboards. Reinstall the delphix-perfdiag package to restore it."
+	else
+		# Interpolate DELPHIX_SYSTEM_UUID / DELPHIX_SYSTEM_VERSION into the
+		# processor's starlark source. Telegraf's own ${VAR} substitution does
+		# not reach inside triple-quoted TOML strings, so we do it here.
+		envsubst '${DELPHIX_SYSTEM_UUID} ${DELPHIX_SYSTEM_VERSION}' < $FLUENTD_PROCESSOR >> $TELEGRAF_CONFIG
+	fi
+	cat $FLUENTD_OUTPUT >> $TELEGRAF_CONFIG
+	has_output=true
+elif fluentd_is_enabled; then
+	logger -t delphix-telegraf "WARNING: FLUENTD_ENABLED is set but $FLUENTD_OUTPUT is missing — metrics will be discarded. Reinstall the delphix-perfdiag package to restore fluentd output."
+fi
+
+if [[ "$has_output" != "true" ]]; then
+	# No output configured. Add discard so Telegraf can start —
 	# it requires at least one output plugin.
 	echo "[[outputs.discard]]" >> $TELEGRAF_CONFIG
 fi

telegraf/perf_fluentd

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Copyright (c) 2026 by Delphix. All rights reserved.
+#
+# Script that enables and disables fluentd 3rd-party metric forwarding for Telegraf.
+#
+
+FLUENTD_FLAG=/etc/telegraf/FLUENTD_ENABLED
+
+function die() {
+	echo -e "$(date +%T:%N:%z): $(basename $0): $*" >&2
+	exit 1
+}
+
+[[ $EUID -ne 0 ]] && die "must be run as root"
+
+function usage() {
+	echo "$(basename $0): $*" >&2
+	echo "Usage: $(basename $0) [enable|disable|status]"
+	exit 2
+}
+
+function enable_fluentd() {
+	date
+	if [[ -f $FLUENTD_FLAG ]]; then
+		echo "fluentd forwarding already enabled — nothing to do"
+		return 0
+	fi
+	echo "Enabling fluentd 3rd-party metric forwarding"
+	touch $FLUENTD_FLAG || die "Failed to create $FLUENTD_FLAG"
+	systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf"
+}
+
+function disable_fluentd() {
+	date
+	if [[ ! -f $FLUENTD_FLAG ]]; then
+		echo "fluentd forwarding already disabled — nothing to do"
+		return 0
+	fi
+	echo "Disabling fluentd 3rd-party metric forwarding"
+	rm -f $FLUENTD_FLAG
+	systemctl restart delphix-telegraf || die "Failed to restart delphix-telegraf"
+}
+
+function status_fluentd() {
+	if [[ -f $FLUENTD_FLAG ]]; then
+		echo "fluentd forwarding: enabled"
+	else
+		echo "fluentd forwarding: disabled"
+	fi
+	echo -n "delphix-telegraf: "
+	systemctl is-active delphix-telegraf
+}
+
+if [[ $# -ne 1 ]]; then
+	usage
+fi
+
+case "$1" in
+enable)  enable_fluentd ;;
+disable) disable_fluentd ;;
+status)  status_fluentd ;;
+*) usage ;;
+esac

telegraf/telegraf.outputs.fluentd

@@ -0,0 +1,22 @@
+# Telegraf HTTP output → local fluentd HTTP source.
+# Forwards performance analytics metrics to fluentd, which routes them to
+# third-party monitoring systems (Splunk, Datadog) via existing
+# <match delphix.metrics.**> rules in fluent.conf.stg.
+# Included by delphix-telegraf-service when FLUENTD_ENABLED flag exists.
+[[outputs.http]]
+  url = "http://localhost:9880/delphix.metrics.platform"
+  method = "POST"
+  data_format = "splunkmetric"
+  splunkmetric_hec_routing = true
+  # One POST per metric: batched splunkmetric emits concatenated JSON with no delimiter, which fluentd's JSON parser silently truncates to the first object.
+  use_batch_format = false
+  flush_interval = "60s"
+  timeout = "10s"
+  # Buffer a few minutes of compat metrics so a brief fluentd restart doesn't
+  # lose data. The processor emits ~6 metrics per flush (one per Java metric
+  # name), so 2048 buffers > 5 hours of history — well above typical downtime.
+  metric_buffer_limit = 2048
+  namepass = ["system.cpu.util", "system.net.total", "system.disk.ops",
+              "system.nfs.ops", "system.iscsi.ops", "system.tcp.total"]
+  [outputs.http.headers]
+    Content-Type = "application/json"