From 3da324e3b58610f9dd6d5db8155b1165db3987e8 Mon Sep 17 00:00:00 2001 From: David Friehs Date: Tue, 15 Sep 2026 20:52:45 +0200 Subject: [PATCH 1/3] model: calculate split states for attn_qkv from n_head * n_embd_head_k required for gemma4 with --fuse-qkv, where n_embd is 5376 but Q is 8192. --- src/llama-model.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3b2536283c57..bf03f8cf29e9 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -648,9 +648,9 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) { - const int64_t n_embd = hparams.n_embd; + const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il); const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il); - GGML_ASSERT(hparams.n_embd_k_gqa() == n_embd_gqa); + GGML_ASSERT(hparams.n_embd_k_gqa(il) == n_embd_gqa); GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); return {{n_embd, 1}, {n_embd_gqa, 2}}; } From b1a9fc96c3bafe3d8b2bf79cb82097397d783c35 Mon Sep 17 00:00:00 2001 From: David Friehs Date: Tue, 15 Sep 2026 20:53:30 +0200 Subject: [PATCH 2/3] model: handle fused full attention layers for qwen35/qwen35moe --- src/llama-model.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index bf03f8cf29e9..bcb024a8adb6 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -599,6 +599,16 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str auto get_split_segments = [&](int axis, uint32_t il) -> std::vector> { if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { + + // fused full attention layers: + if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) { + const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il) * 2; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il); + GGML_ASSERT(hparams.n_embd_k_gqa(il) == n_embd_gqa); + GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); + return {{n_embd, 1}, {n_embd_gqa, 2}}; + } + const int64_t head_k_dim = hparams.ssm_d_state; const int64_t head_v_dim = hparams.ssm_d_state; const int64_t n_k_heads = hparams.ssm_n_group; @@ -763,6 +773,11 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) { GGML_ASSERT(segments.size() == 2); + // fused full attention layers need Q gate tensors handled like above: + if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || + ud->model->arch == LLM_ARCH_QWEN4EXP) { + return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv}; + } return {granularity_q, granularity_kv}; } } From c2651dc8a5577ffeee96a0ab7c41cab47eeffd2f Mon Sep 17 00:00:00 2001 From: David Friehs Date: Wed, 16 Sep 2026 20:39:20 +0200 Subject: [PATCH 3/3] model: add TODO: [TAG_SPLIT_QGATE_QWEN] --- src/llama-model.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index bcb024a8adb6..62d0d944c375 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -597,10 +597,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str }; auto get_split_segments = [&](int axis, uint32_t il) -> std::vector> { + // TODO: clarify why this is necessary specifically for these models + // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { - // fused full attention layers: + // fused full attention layers with Q gate tensors that need n_embd doubled: if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) { const int64_t n_embd = hparams.n_head(il) * hparams.n_embd_head_k(il) * 2; const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il); @@ -746,6 +748,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) { GGML_ASSERT(segments.size() == 1); // some models have Q gate tensors, for those cases the granularity needs to be doubled: + // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { return {std::lcm(2*n_embd_q, blck_size_perf)}; @@ -774,6 +777,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) { GGML_ASSERT(segments.size() == 2); // fused full attention layers need Q gate tensors handled like above: + // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv};