From c03dc68bb8761a90eca045160e90b419c9b0dded Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 31 Aug 2026 02:58:30 +0000 Subject: [PATCH 1/2] models: use flash-linear-attention's l2norm for gated delta net q/k The GDN q/k normalization is defined by flash-linear-attention as l2norm(x) = x * rsqrt(sum(x*x) + eps) with eps inside the root. Every GDN call site in the tree uses ggml_l2_norm instead, which is x / max(sqrt(sum(x*x)), eps), i.e. torch.nn.functional.normalize - its CUDA kernel cites that page. The clamp never engages at these magnitudes, so in practice llama.cpp normalizes with no epsilon at all where the reference has one inside the root. transformers made the same substitution when it first added Qwen3-Next and corrected it three days later in huggingface/transformers#40842, 'Fix the misalignment between the l2norm in GDN of Qwen3-Next and the implementation in the FLA library'. vLLM and SGLang vendor FLA rather than reimplementing it, so neither ever had the clamp. eps keeps coming from the checkpoint, exactly as every call site already passed it. The references hardcode 1e-6 for this norm; that is a separate question and the two agree on every GDN checkpoint in the wild. ggml_l2_norm itself is correct and unchanged, as is rwkv7-base, its original caller, which passes normalize's own default eps of 1e-12. No new ggml op: rms_norm already carries eps inside the root, so rms_norm(x, eps/n) * (1/sqrt(n)) is exactly x * rsqrt(sum(x*x) + eps). --- src/models/bailingmoe3.cpp | 4 ++-- src/models/kimi-k3.cpp | 6 +++--- src/models/kimi-linear.cpp | 5 +++-- src/models/models.h | 6 ++++++ src/models/qwen35.cpp | 5 +++-- src/models/qwen35moe.cpp | 5 +++-- src/models/qwen3next.cpp | 5 +++-- src/models/qwen4exp.cpp | 5 +++-- 8 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index 0637931cc0c..abc4440a2fe 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -281,8 +281,8 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur); beta = ggml_sigmoid(ctx0, ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs)); - q = ggml_l2_norm(ctx0, q, hparams.f_norm_rms_eps); - k = ggml_l2_norm(ctx0, k, hparams.f_norm_rms_eps); + q = build_gdn_l2_norm(ctx0, q, hparams.f_norm_rms_eps); + k = build_gdn_l2_norm(ctx0, k, hparams.f_norm_rms_eps); ggml_tensor * states_all = mctx_cur->get_s_l(il); ggml_tensor * state = build_rs(inp_rs, states_all, hparams.n_embd_s(), n_seqs); diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index d952d72cdf1..6d09ed00e06 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -441,9 +441,9 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head_kda, n_seqs); - const float eps = hparams.f_norm_rms_eps; - Qcur = ggml_l2_norm(ctx0, Qcur, eps); - Kcur = ggml_l2_norm(ctx0, Kcur, eps); + const float eps_norm = hparams.f_norm_rms_eps; + Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); + Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp index 367f6990d1f..69962f220f8 100644 --- a/src/models/kimi-linear.cpp +++ b/src/models/kimi-linear.cpp @@ -331,10 +331,11 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs); + const float eps_norm = hparams.f_norm_rms_eps; - Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm); - Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm); + Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); + Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); diff --git a/src/models/models.h b/src/models/models.h index 9b87a40d5af..76b017e0298 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -10,6 +10,12 @@ class llama_memory_hybrid_idx_context; +static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) { + const float n = x->ne[0]; + + return ggml_scale(ctx, ggml_rms_norm(ctx, x, eps/n), 1.0f/sqrtf(n)); +} + // // base classes // diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 309dd432447..f6a71672c4d 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -427,10 +427,11 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 38f2a57985a..988cbd6ccd7 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -451,10 +451,11 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index 0808fd87aa0..783289f4d1d 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -507,10 +507,11 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index abf6a0502fb..4c9ef69a284 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -875,10 +875,11 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); // repeat to match shapes when head keys != value keys; unneeded with the fused GDN if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { From 757abc99b3afb357bc40da9d1fa97c2d2ac10028 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 31 Aug 2026 00:12:52 -0700 Subject: [PATCH 2/2] Update src/models/models.h Co-authored-by: Georgi Gerganov --- src/models/models.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/models.h b/src/models/models.h index 76b017e0298..29e56d88118 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -10,6 +10,7 @@ class llama_memory_hybrid_idx_context; +// ref: https://github.com/ggml-org/llama.cpp/pull/28068 static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) { const float n = x->ne[0];