diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index 0637931cc0c..abc4440a2fe 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -281,8 +281,8 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur); beta = ggml_sigmoid(ctx0, ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs)); - q = ggml_l2_norm(ctx0, q, hparams.f_norm_rms_eps); - k = ggml_l2_norm(ctx0, k, hparams.f_norm_rms_eps); + q = build_gdn_l2_norm(ctx0, q, hparams.f_norm_rms_eps); + k = build_gdn_l2_norm(ctx0, k, hparams.f_norm_rms_eps); ggml_tensor * states_all = mctx_cur->get_s_l(il); ggml_tensor * state = build_rs(inp_rs, states_all, hparams.n_embd_s(), n_seqs); diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index d952d72cdf1..6d09ed00e06 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -441,9 +441,9 @@ ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer( ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head_kda, n_seqs); - const float eps = hparams.f_norm_rms_eps; - Qcur = ggml_l2_norm(ctx0, Qcur, eps); - Kcur = ggml_l2_norm(ctx0, Kcur, eps); + const float eps_norm = hparams.f_norm_rms_eps; + Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); + Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp index 367f6990d1f..69962f220f8 100644 --- a/src/models/kimi-linear.cpp +++ b/src/models/kimi-linear.cpp @@ -331,10 +331,11 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs); + const float eps_norm = hparams.f_norm_rms_eps; - Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm); - Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm); + Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); + Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); diff --git a/src/models/models.h b/src/models/models.h index 9b87a40d5af..29e56d88118 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -10,6 +10,13 @@ class llama_memory_hybrid_idx_context; +// ref: https://github.com/ggml-org/llama.cpp/pull/28068 +static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) { + const float n = x->ne[0]; + + return ggml_scale(ctx, ggml_rms_norm(ctx, x, eps/n), 1.0f/sqrtf(n)); +} + // // base classes // diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 309dd432447..f6a71672c4d 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -427,10 +427,11 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 38f2a57985a..988cbd6ccd7 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -451,10 +451,11 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index 0808fd87aa0..783289f4d1d 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -507,10 +507,11 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index abf6a0502fb..4c9ef69a284 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -875,10 +875,11 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( cb(k_conv, "k_conv", il); cb(v_conv, "v_conv", il); + const float eps_norm = hparams.f_norm_rms_eps; - q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm); - k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm); + q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); + k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); // repeat to match shapes when head keys != value keys; unneeded with the fused GDN if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {